^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * preemptoff and irqoff tracepoints
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/kallsyms.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #define CREATE_TRACE_POINTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <trace/events/preemptirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #undef CREATE_TRACE_POINTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <trace/hooks/preemptirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #ifdef CONFIG_TRACE_IRQFLAGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) /* Per-cpu variable to prevent redundant calls when IRQs already off */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static DEFINE_PER_CPU(int, tracing_irq_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * Like trace_hardirqs_on() but without the lockdep invocation. This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * used in the low level entry code where the ordering vs. RCU is important
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * and lockdep uses a staged approach which splits the lockdep hardirq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * tracking into a RCU on and a RCU off section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) void trace_hardirqs_on_prepare(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) if (this_cpu_read(tracing_irq_cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) trace_android_rvh_irqs_enable(CALLER_ADDR0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) this_cpu_write(tracing_irq_cpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) EXPORT_SYMBOL(trace_hardirqs_on_prepare);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) void trace_hardirqs_on(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) if (this_cpu_read(tracing_irq_cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) trace_android_rvh_irqs_enable(CALLER_ADDR0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) this_cpu_write(tracing_irq_cpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) lockdep_hardirqs_on_prepare(CALLER_ADDR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) lockdep_hardirqs_on(CALLER_ADDR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) EXPORT_SYMBOL(trace_hardirqs_on);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) NOKPROBE_SYMBOL(trace_hardirqs_on);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * Like trace_hardirqs_off() but without the lockdep invocation. This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * used in the low level entry code where the ordering vs. RCU is important
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * and lockdep uses a staged approach which splits the lockdep hardirq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * tracking into a RCU on and a RCU off section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) void trace_hardirqs_off_finish(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) if (!this_cpu_read(tracing_irq_cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) this_cpu_write(tracing_irq_cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) trace_android_rvh_irqs_disable(CALLER_ADDR0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) EXPORT_SYMBOL(trace_hardirqs_off_finish);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) void trace_hardirqs_off(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) lockdep_hardirqs_off(CALLER_ADDR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (!this_cpu_read(tracing_irq_cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) this_cpu_write(tracing_irq_cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) trace_android_rvh_irqs_disable(CALLER_ADDR0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) CALLER_ADDR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) EXPORT_SYMBOL(trace_hardirqs_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) NOKPROBE_SYMBOL(trace_hardirqs_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (this_cpu_read(tracing_irq_cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) trace_android_rvh_irqs_enable(CALLER_ADDR0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) caller_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) this_cpu_write(tracing_irq_cpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) lockdep_hardirqs_on_prepare(CALLER_ADDR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) lockdep_hardirqs_on(CALLER_ADDR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) EXPORT_SYMBOL(trace_hardirqs_on_caller);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) NOKPROBE_SYMBOL(trace_hardirqs_on_caller);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) lockdep_hardirqs_off(CALLER_ADDR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (!this_cpu_read(tracing_irq_cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) this_cpu_write(tracing_irq_cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) trace_android_rvh_irqs_enable(CALLER_ADDR0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) caller_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) EXPORT_SYMBOL(trace_hardirqs_off_caller);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #endif /* CONFIG_TRACE_IRQFLAGS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) void trace_preempt_on(unsigned long a0, unsigned long a1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) trace_preempt_enable_rcuidle(a0, a1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) trace_android_rvh_preempt_enable(a0, a1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) tracer_preempt_on(a0, a1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) void trace_preempt_off(unsigned long a0, unsigned long a1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) trace_preempt_disable_rcuidle(a0, a1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) trace_android_rvh_preempt_disable(a0, a1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) tracer_preempt_off(a0, a1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags