^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * trace task wakeup timings
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Based on code from the latency_tracer, that is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Copyright (C) 2004-2006 Ingo Molnar
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Copyright (C) 2004 Nadia Yvette Chambers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/kallsyms.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched/rt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/sched/deadline.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <trace/events/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static struct trace_array *wakeup_trace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) static int __read_mostly tracer_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) static struct task_struct *wakeup_task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static int wakeup_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static int wakeup_current_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static unsigned wakeup_prio = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) static int wakeup_rt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static int wakeup_dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) static int tracing_dl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) static arch_spinlock_t wakeup_lock =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static void wakeup_reset(struct trace_array *tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) static void __wakeup_reset(struct trace_array *tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static int start_func_tracer(struct trace_array *tr, int graph);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static void stop_func_tracer(struct trace_array *tr, int graph);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static int save_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) # define is_graph(tr) false
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #ifdef CONFIG_FUNCTION_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static bool function_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * Prologue for the wakeup function tracers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * Returns 1 if it is OK to continue, and preemption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * is disabled and data->disabled is incremented.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * 0 if the trace is to be ignored, and preemption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * is not disabled and data->disabled is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * kept the same.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * Note, this function is also used outside this ifdef but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * inside the #ifdef of the function graph tracer below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * This is OK, since the function graph tracer is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * dependent on the function tracer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) func_prolog_preempt_disable(struct trace_array *tr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct trace_array_cpu **data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) int *pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) long disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (likely(!wakeup_task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) *pc = preempt_count();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) preempt_disable_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (cpu != wakeup_current_cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) goto out_enable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) *data = per_cpu_ptr(tr->array_buffer.data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) disabled = atomic_inc_return(&(*data)->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (unlikely(disabled != 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) atomic_dec(&(*data)->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) out_enable:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) preempt_enable_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static int wakeup_display_graph(struct trace_array *tr, int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (!(is_graph(tr) ^ set))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) stop_func_tracer(tr, !set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) wakeup_reset(wakeup_trace);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) tr->max_latency = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return start_func_tracer(tr, set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct trace_array *tr = wakeup_trace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct trace_array_cpu *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) int pc, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (ftrace_graph_ignore_func(trace))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * Do not trace a function if it's filtered by set_graph_notrace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * Make the index of ret stack negative to indicate that it should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * ignore further functions. But it needs its own ret stack entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * to recover the original index in order to continue tracing after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * returning from the function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (ftrace_graph_notrace_addr(trace->func))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) if (!func_prolog_preempt_disable(tr, &data, &pc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) local_save_flags(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) ret = __trace_graph_entry(tr, trace, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) atomic_dec(&data->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) preempt_enable_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static void wakeup_graph_return(struct ftrace_graph_ret *trace)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct trace_array *tr = wakeup_trace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct trace_array_cpu *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) int pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) ftrace_graph_addr_finish(trace);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (!func_prolog_preempt_disable(tr, &data, &pc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) local_save_flags(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) __trace_graph_return(tr, trace, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) atomic_dec(&data->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) preempt_enable_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static struct fgraph_ops fgraph_wakeup_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) .entryfunc = &wakeup_graph_entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) .retfunc = &wakeup_graph_return,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) static void wakeup_trace_open(struct trace_iterator *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (is_graph(iter->tr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) graph_trace_open(iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static void wakeup_trace_close(struct trace_iterator *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (iter->private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) graph_trace_close(iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) TRACE_GRAPH_PRINT_CPU | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) TRACE_GRAPH_PRINT_REL_TIME | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) TRACE_GRAPH_PRINT_DURATION | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) TRACE_GRAPH_PRINT_OVERHEAD | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) TRACE_GRAPH_PRINT_IRQS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * In graph mode call the graph tracer output function,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * otherwise go with the TRACE_FN event handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (is_graph(iter->tr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return TRACE_TYPE_UNHANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static void wakeup_print_header(struct seq_file *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (is_graph(wakeup_trace))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) trace_default_header(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) #endif /* else CONFIG_FUNCTION_GRAPH_TRACER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * wakeup uses its own tracer function to keep the overhead down:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) struct ftrace_ops *op, struct pt_regs *pt_regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct trace_array *tr = wakeup_trace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct trace_array_cpu *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) int pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (!func_prolog_preempt_disable(tr, &data, &pc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) trace_function(tr, ip, parent_ip, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) atomic_dec(&data->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) preempt_enable_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) static int register_wakeup_function(struct trace_array *tr, int graph, int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (graph)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) ret = register_ftrace_graph(&fgraph_wakeup_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) ret = register_ftrace_function(tr->ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) function_enabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) static void unregister_wakeup_function(struct trace_array *tr, int graph)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (!function_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (graph)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) unregister_ftrace_graph(&fgraph_wakeup_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) unregister_ftrace_function(tr->ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) function_enabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (!(mask & TRACE_ITER_FUNCTION))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) register_wakeup_function(tr, is_graph(tr), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) unregister_wakeup_function(tr, is_graph(tr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #else /* CONFIG_FUNCTION_TRACER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) static int register_wakeup_function(struct trace_array *tr, int graph, int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static void unregister_wakeup_function(struct trace_array *tr, int graph) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) #endif /* else CONFIG_FUNCTION_TRACER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) #ifndef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return TRACE_TYPE_UNHANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static void wakeup_trace_open(struct trace_iterator *iter) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) static void wakeup_trace_close(struct trace_iterator *iter) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) static void wakeup_print_header(struct seq_file *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) trace_default_header(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #endif /* !CONFIG_FUNCTION_GRAPH_TRACER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) __trace_function(struct trace_array *tr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) unsigned long ip, unsigned long parent_ip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) unsigned long flags, int pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (is_graph(tr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) trace_graph_function(tr, ip, parent_ip, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) trace_function(tr, ip, parent_ip, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct tracer *tracer = tr->current_trace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (wakeup_function_set(tr, mask, set))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (mask & TRACE_ITER_DISPLAY_GRAPH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) return wakeup_display_graph(tr, set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) return trace_keep_overwrite(tracer, mask, set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) static int start_func_tracer(struct trace_array *tr, int graph)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) ret = register_wakeup_function(tr, graph, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (!ret && tracing_is_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) tracer_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) tracer_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) static void stop_func_tracer(struct trace_array *tr, int graph)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) tracer_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) unregister_wakeup_function(tr, graph);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * Should this new latency be reported/recorded?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static bool report_latency(struct trace_array *tr, u64 delta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) if (tracing_thresh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (delta < tracing_thresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) if (delta <= tr->max_latency)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (task != wakeup_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) wakeup_current_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) tracing_sched_switch_trace(struct trace_array *tr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) struct task_struct *prev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) struct task_struct *next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) unsigned long flags, int pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) struct trace_event_call *call = &event_context_switch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct trace_buffer *buffer = tr->array_buffer.buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) struct ring_buffer_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct ctx_switch_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) event = trace_buffer_lock_reserve(buffer, TRACE_CTX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) sizeof(*entry), flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (!event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) entry = ring_buffer_event_data(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) entry->prev_pid = prev->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) entry->prev_prio = prev->prio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) entry->prev_state = task_state_index(prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) entry->next_pid = next->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) entry->next_prio = next->prio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) entry->next_state = task_state_index(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) entry->next_cpu = task_cpu(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (!call_filter_check_discard(call, entry, buffer, event))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) trace_buffer_unlock_commit(tr, buffer, event, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) tracing_sched_wakeup_trace(struct trace_array *tr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) struct task_struct *wakee,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) struct task_struct *curr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) unsigned long flags, int pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) struct trace_event_call *call = &event_wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) struct ring_buffer_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) struct ctx_switch_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) struct trace_buffer *buffer = tr->array_buffer.buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) event = trace_buffer_lock_reserve(buffer, TRACE_WAKE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) sizeof(*entry), flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (!event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) entry = ring_buffer_event_data(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) entry->prev_pid = curr->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) entry->prev_prio = curr->prio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) entry->prev_state = task_state_index(curr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) entry->next_pid = wakee->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) entry->next_prio = wakee->prio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) entry->next_state = task_state_index(wakee);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) entry->next_cpu = task_cpu(wakee);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (!call_filter_check_discard(call, entry, buffer, event))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) trace_buffer_unlock_commit(tr, buffer, event, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) static void notrace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) probe_wakeup_sched_switch(void *ignore, bool preempt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) struct task_struct *prev, struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) struct trace_array_cpu *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) u64 T0, T1, delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) long disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) int pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) tracing_record_cmdline(prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (unlikely(!tracer_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * When we start a new trace, we set wakeup_task to NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * and then set tracer_enabled = 1. We want to make sure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * that another CPU does not see the tracer_enabled = 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * and the wakeup_task with an older task, that might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * actually be the same as next.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) if (next != wakeup_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) pc = preempt_count();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /* disable local data, not wakeup_cpu data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) if (likely(disabled != 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) arch_spin_lock(&wakeup_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) /* We could race with grabbing wakeup_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (unlikely(!tracer_enabled || next != wakeup_task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) /* The task we are waiting for is waking up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) __trace_stack(wakeup_trace, flags, 0, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) T0 = data->preempt_timestamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) T1 = ftrace_now(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) delta = T1-T0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (!report_latency(wakeup_trace, delta))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (likely(!is_tracing_stopped())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) wakeup_trace->max_latency = delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) __wakeup_reset(wakeup_trace);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) arch_spin_unlock(&wakeup_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) static void __wakeup_reset(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) wakeup_cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) wakeup_prio = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) tracing_dl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (wakeup_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) put_task_struct(wakeup_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) wakeup_task = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) static void wakeup_reset(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) tracing_reset_online_cpus(&tr->array_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) arch_spin_lock(&wakeup_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) __wakeup_reset(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) arch_spin_unlock(&wakeup_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) probe_wakeup(void *ignore, struct task_struct *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) struct trace_array_cpu *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) long disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) int pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (likely(!tracer_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) tracing_record_cmdline(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) tracing_record_cmdline(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) * Semantic is like this:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * - wakeup tracer handles all tasks in the system, independently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * from their scheduling class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) * - wakeup_rt tracer handles tasks belonging to sched_dl and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * sched_rt class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * - wakeup_dl handles tasks belonging to sched_dl class only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (tracing_dl || (wakeup_dl && !dl_task(p)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) (wakeup_rt && !dl_task(p) && !rt_task(p)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) pc = preempt_count();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (unlikely(disabled != 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) /* interrupts should be off from try_to_wake_up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) arch_spin_lock(&wakeup_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /* check for races. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (!tracer_enabled || tracing_dl ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) (!dl_task(p) && p->prio >= wakeup_prio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) goto out_locked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) /* reset the trace */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) __wakeup_reset(wakeup_trace);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) wakeup_cpu = task_cpu(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) wakeup_current_cpu = wakeup_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) wakeup_prio = p->prio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * Once you start tracing a -deadline task, don't bother tracing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * another task until the first one wakes up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) if (dl_task(p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) tracing_dl = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) tracing_dl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) wakeup_task = get_task_struct(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) local_save_flags(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) data->preempt_timestamp = ftrace_now(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) __trace_stack(wakeup_trace, flags, 0, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * We must be careful in using CALLER_ADDR2. But since wake_up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * is not called by an assembly function (where as schedule is)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * it should be safe to use it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) out_locked:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) arch_spin_unlock(&wakeup_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) static void start_wakeup_tracer(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) ret = register_trace_sched_wakeup(probe_wakeup, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) pr_info("wakeup trace: Couldn't activate tracepoint"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) " probe to kernel_sched_wakeup\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) ret = register_trace_sched_wakeup_new(probe_wakeup, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) pr_info("wakeup trace: Couldn't activate tracepoint"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) " probe to kernel_sched_wakeup_new\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) goto fail_deprobe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) pr_info("sched trace: Couldn't activate tracepoint"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) " probe to kernel_sched_switch\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) goto fail_deprobe_wake_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) pr_info("wakeup trace: Couldn't activate tracepoint"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) " probe to kernel_sched_migrate_task\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) goto fail_deprobe_sched_switch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) wakeup_reset(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * Don't let the tracer_enabled = 1 show up before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) * the wakeup_task is reset. This may be overkill since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) * wakeup_reset does a spin_unlock after setting the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) * wakeup_task to NULL, but I want to be safe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * This is a slow path anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (start_func_tracer(tr, is_graph(tr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) printk(KERN_ERR "failed to start wakeup tracer\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) fail_deprobe_sched_switch:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) fail_deprobe_wake_new:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) fail_deprobe:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) unregister_trace_sched_wakeup(probe_wakeup, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) static void stop_wakeup_tracer(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) tracer_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) stop_func_tracer(tr, is_graph(tr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) unregister_trace_sched_wakeup(probe_wakeup, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) static bool wakeup_busy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) static int __wakeup_tracer_init(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) save_flags = tr->trace_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) /* non overwrite screws up the latency tracers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) tr->max_latency = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) wakeup_trace = tr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) ftrace_init_array_ops(tr, wakeup_tracer_call);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) start_wakeup_tracer(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) wakeup_busy = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) static int wakeup_tracer_init(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) if (wakeup_busy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) wakeup_dl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) wakeup_rt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) return __wakeup_tracer_init(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) static int wakeup_rt_tracer_init(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (wakeup_busy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) wakeup_dl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) wakeup_rt = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) return __wakeup_tracer_init(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) static int wakeup_dl_tracer_init(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (wakeup_busy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) wakeup_dl = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) wakeup_rt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) return __wakeup_tracer_init(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) static void wakeup_tracer_reset(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) stop_wakeup_tracer(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) /* make sure we put back any tasks we are tracing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) wakeup_reset(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) ftrace_reset_array_ops(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) wakeup_busy = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) static void wakeup_tracer_start(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) wakeup_reset(tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) tracer_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) static void wakeup_tracer_stop(struct trace_array *tr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) tracer_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) static struct tracer wakeup_tracer __read_mostly =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) .name = "wakeup",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) .init = wakeup_tracer_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) .reset = wakeup_tracer_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) .start = wakeup_tracer_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) .stop = wakeup_tracer_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) .print_max = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) .print_header = wakeup_print_header,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) .print_line = wakeup_print_line,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) .flag_changed = wakeup_flag_changed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) #ifdef CONFIG_FTRACE_SELFTEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) .selftest = trace_selftest_startup_wakeup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) .open = wakeup_trace_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) .close = wakeup_trace_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) .allow_instances = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) .use_max_tr = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) static struct tracer wakeup_rt_tracer __read_mostly =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) .name = "wakeup_rt",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) .init = wakeup_rt_tracer_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) .reset = wakeup_tracer_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) .start = wakeup_tracer_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) .stop = wakeup_tracer_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) .print_max = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) .print_header = wakeup_print_header,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) .print_line = wakeup_print_line,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) .flag_changed = wakeup_flag_changed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) #ifdef CONFIG_FTRACE_SELFTEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) .selftest = trace_selftest_startup_wakeup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) .open = wakeup_trace_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) .close = wakeup_trace_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) .allow_instances = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) .use_max_tr = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) static struct tracer wakeup_dl_tracer __read_mostly =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) .name = "wakeup_dl",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) .init = wakeup_dl_tracer_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) .reset = wakeup_tracer_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) .start = wakeup_tracer_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) .stop = wakeup_tracer_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) .print_max = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) .print_header = wakeup_print_header,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) .print_line = wakeup_print_line,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) .flag_changed = wakeup_flag_changed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) #ifdef CONFIG_FTRACE_SELFTEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) .selftest = trace_selftest_startup_wakeup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) .open = wakeup_trace_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) .close = wakeup_trace_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) .allow_instances = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) .use_max_tr = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) __init static int init_wakeup_tracer(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) ret = register_tracer(&wakeup_tracer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) ret = register_tracer(&wakeup_rt_tracer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) ret = register_tracer(&wakeup_dl_tracer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) core_initcall(init_wakeup_tracer);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags