^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * unlikely profiler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/kallsyms.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/irqflags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/local.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "trace_stat.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "trace_output.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #ifdef CONFIG_BRANCH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static struct tracer branch_trace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) static int branch_tracing_enabled __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static DEFINE_MUTEX(branch_tracing_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static struct trace_array *branch_tracer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) probe_likely_condition(struct ftrace_likely_data *f, int val, int expect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct trace_event_call *call = &event_branch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) struct trace_array *tr = branch_tracer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) struct trace_buffer *buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) struct trace_array_cpu *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct ring_buffer_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct trace_branch *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) int pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) const char *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) if (current->trace_recursion & TRACE_BRANCH_BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * I would love to save just the ftrace_likely_data pointer, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * this code can also be used by modules. Ugly things can happen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * if the module is unloaded, and then we go and read the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * pointer. This is slower, but much safer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) if (unlikely(!tr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) raw_local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) current->trace_recursion |= TRACE_BRANCH_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) data = this_cpu_ptr(tr->array_buffer.data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) if (atomic_read(&data->disabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) pc = preempt_count();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) buffer = tr->array_buffer.buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) sizeof(*entry), flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (!event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) entry = ring_buffer_event_data(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /* Strip off the path, only save the file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) p = f->data.file + strlen(f->data.file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) while (p >= f->data.file && *p != '/')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) p--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) p++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) strncpy(entry->func, f->data.func, TRACE_FUNC_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) strncpy(entry->file, p, TRACE_FILE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) entry->func[TRACE_FUNC_SIZE] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) entry->file[TRACE_FILE_SIZE] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) entry->constant = f->constant;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) entry->line = f->data.line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) entry->correct = val == expect;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) if (!call_filter_check_discard(call, entry, buffer, event))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) trace_buffer_unlock_commit_nostack(buffer, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) current->trace_recursion &= ~TRACE_BRANCH_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) raw_local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) if (!branch_tracing_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) probe_likely_condition(f, val, expect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) int enable_branch_tracing(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) mutex_lock(&branch_tracing_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) branch_tracer = tr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * Must be seen before enabling. The reader is a condition
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * where we do not need a matching rmb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) branch_tracing_enabled++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) mutex_unlock(&branch_tracing_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) void disable_branch_tracing(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) mutex_lock(&branch_tracing_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (!branch_tracing_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) branch_tracing_enabled--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) mutex_unlock(&branch_tracing_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static int branch_trace_init(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) return enable_branch_tracing(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void branch_trace_reset(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) disable_branch_tracing();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static enum print_line_t trace_branch_print(struct trace_iterator *iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int flags, struct trace_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct trace_branch *field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) trace_assign_type(field, iter->ent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) field->correct ? " ok " : " MISS ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) field->func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) field->file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) field->line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return trace_handle_return(&iter->seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static void branch_print_header(struct seq_file *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) seq_puts(s, "# TASK-PID CPU# TIMESTAMP CORRECT"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) " FUNC:FILE:LINE\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) "# | | | | | "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) " |\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static struct trace_event_functions trace_branch_funcs = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) .trace = trace_branch_print,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static struct trace_event trace_branch_event = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) .type = TRACE_BRANCH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) .funcs = &trace_branch_funcs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static struct tracer branch_trace __read_mostly =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) .name = "branch",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) .init = branch_trace_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) .reset = branch_trace_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #ifdef CONFIG_FTRACE_SELFTEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) .selftest = trace_selftest_startup_branch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) #endif /* CONFIG_FTRACE_SELFTEST */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) .print_header = branch_print_header,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) __init static int init_branch_tracer(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) ret = register_trace_event(&trace_branch_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) printk(KERN_WARNING "Warning: could not register "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) "branch events\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return register_tracer(&branch_trace);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) core_initcall(init_branch_tracer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #endif /* CONFIG_BRANCH_TRACER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) void ftrace_likely_update(struct ftrace_likely_data *f, int val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) int expect, int is_constant)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) unsigned long flags = user_access_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /* A constant is always correct */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) if (is_constant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) f->constant++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) val = expect;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * I would love to have a trace point here instead, but the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * trace point code is so inundated with unlikely and likely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * conditions that the recursive nightmare that exists is too
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * much to try to get working. At least for now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) trace_likely_condition(f, val, expect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) /* FIXME: Make this atomic! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) if (val == expect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) f->data.correct++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) f->data.incorrect++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) user_access_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) EXPORT_SYMBOL(ftrace_likely_update);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) extern unsigned long __start_annotated_branch_profile[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) extern unsigned long __stop_annotated_branch_profile[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) static int annotated_branch_stat_headers(struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) seq_puts(m, " correct incorrect % "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) " Function "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) " File Line\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) " ------- --------- - "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) " -------- "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) " ---- ----\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) static inline long get_incorrect_percent(const struct ftrace_branch_data *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) long percent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (p->correct) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) percent = p->incorrect * 100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) percent /= p->correct + p->incorrect;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) percent = p->incorrect ? 100 : -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return percent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static const char *branch_stat_process_file(struct ftrace_branch_data *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) const char *f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) /* Only print the file, not the path */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) f = p->file + strlen(p->file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) while (f >= p->file && *f != '/')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) f--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return ++f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static void branch_stat_show(struct seq_file *m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) struct ftrace_branch_data *p, const char *f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) long percent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * The miss is overlayed on correct, and hit on incorrect.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) percent = get_incorrect_percent(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (percent < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) seq_puts(m, " X ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) seq_printf(m, "%3ld ", percent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static int branch_stat_show_normal(struct seq_file *m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) struct ftrace_branch_data *p, const char *f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) branch_stat_show(m, p, f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) static int annotate_branch_stat_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) struct ftrace_likely_data *p = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) const char *f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) int l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) f = branch_stat_process_file(&p->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (!p->constant)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) return branch_stat_show_normal(m, &p->data, f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) l = snprintf(NULL, 0, "/%lu", p->constant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) l = l > 8 ? 0 : 8 - l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) seq_printf(m, "%8lu/%lu %*lu ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) p->data.correct, p->constant, l, p->data.incorrect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) branch_stat_show(m, &p->data, f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static void *annotated_branch_stat_start(struct tracer_stat *trace)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) return __start_annotated_branch_profile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) static void *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) annotated_branch_stat_next(void *v, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) struct ftrace_likely_data *p = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) ++p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if ((void *)p >= (void *)__stop_annotated_branch_profile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static int annotated_branch_stat_cmp(const void *p1, const void *p2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) const struct ftrace_branch_data *a = p1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) const struct ftrace_branch_data *b = p2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) long percent_a, percent_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) percent_a = get_incorrect_percent(a);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) percent_b = get_incorrect_percent(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) if (percent_a < percent_b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) if (percent_a > percent_b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (a->incorrect < b->incorrect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (a->incorrect > b->incorrect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * Since the above shows worse (incorrect) cases
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * first, we continue that by showing best (correct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * cases last.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) if (a->correct > b->correct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (a->correct < b->correct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) static struct tracer_stat annotated_branch_stats = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) .name = "branch_annotated",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) .stat_start = annotated_branch_stat_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) .stat_next = annotated_branch_stat_next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) .stat_cmp = annotated_branch_stat_cmp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) .stat_headers = annotated_branch_stat_headers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) .stat_show = annotate_branch_stat_show
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) __init static int init_annotated_branch_stats(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) ret = register_stat_tracer(&annotated_branch_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) printk(KERN_WARNING "Warning: could not register "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) "annotated branches stats\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) fs_initcall(init_annotated_branch_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) #ifdef CONFIG_PROFILE_ALL_BRANCHES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) extern unsigned long __start_branch_profile[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) extern unsigned long __stop_branch_profile[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) static int all_branch_stat_headers(struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) seq_puts(m, " miss hit % "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) " Function "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) " File Line\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) " ------- --------- - "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) " -------- "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) " ---- ----\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static void *all_branch_stat_start(struct tracer_stat *trace)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return __start_branch_profile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) static void *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) all_branch_stat_next(void *v, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct ftrace_branch_data *p = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) ++p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if ((void *)p >= (void *)__stop_branch_profile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) return p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) static int all_branch_stat_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) struct ftrace_branch_data *p = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) const char *f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) f = branch_stat_process_file(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) return branch_stat_show_normal(m, p, f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) static struct tracer_stat all_branch_stats = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) .name = "branch_all",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) .stat_start = all_branch_stat_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) .stat_next = all_branch_stat_next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) .stat_headers = all_branch_stat_headers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) .stat_show = all_branch_stat_show
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) __init static int all_annotated_branch_stats(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) ret = register_stat_tracer(&all_branch_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) printk(KERN_WARNING "Warning: could not register "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) "all branches stats\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) fs_initcall(all_annotated_branch_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) #endif /* CONFIG_PROFILE_ALL_BRANCHES */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags