^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * RCU CPU stall warnings for normal RCU grace periods
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright IBM Corporation, 2019
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Paul E. McKenney <paulmck@linux.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/kvm_para.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) //////////////////////////////////////////////////////////////////////////////
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) //
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) // Controlling CPU stall warnings, including delay calculation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) /* panic() on RCU Stall sysctl. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) int sysctl_panic_on_rcu_stall __read_mostly = CONFIG_BOOTPARAM_RCU_STALL_PANIC_VALUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) ATOMIC_NOTIFIER_HEAD(rcu_stall_notifier_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #ifdef CONFIG_PROVE_RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define RCU_STALL_DELAY_DELTA (5 * HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define RCU_STALL_DELAY_DELTA 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define RCU_STALL_MIGHT_DIV 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define RCU_STALL_MIGHT_MIN (2 * HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /* Limit-check stall timeouts specified at boottime and runtime. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) int rcu_jiffies_till_stall_check(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) int till_stall_check = READ_ONCE(rcu_cpu_stall_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * Limit check must be consistent with the Kconfig limits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * for CONFIG_RCU_CPU_STALL_TIMEOUT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) if (till_stall_check < 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) WRITE_ONCE(rcu_cpu_stall_timeout, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) till_stall_check = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) } else if (till_stall_check > 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) WRITE_ONCE(rcu_cpu_stall_timeout, 300);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) till_stall_check = 300;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) EXPORT_SYMBOL_GPL(rcu_jiffies_till_stall_check);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * rcu_gp_might_be_stalled - Is it likely that the grace period is stalled?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * Returns @true if the current grace period is sufficiently old that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * it is reasonable to assume that it might be stalled. This can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * useful when deciding whether to allocate memory to enable RCU-mediated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * freeing on the one hand or just invoking synchronize_rcu() on the other.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * The latter is preferable when the grace period is stalled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * Note that sampling of the .gp_start and .gp_seq fields must be done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * carefully to avoid false positives at the beginnings and ends of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * grace periods.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) bool rcu_gp_might_be_stalled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) unsigned long d = rcu_jiffies_till_stall_check() / RCU_STALL_MIGHT_DIV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) unsigned long j = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (d < RCU_STALL_MIGHT_MIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) d = RCU_STALL_MIGHT_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) smp_mb(); // jiffies before .gp_seq to avoid false positives.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) if (!rcu_gp_in_progress())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) // Long delays at this point avoids false positive, but a delay
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) // of ULONG_MAX/4 jiffies voids your no-false-positive warranty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) smp_mb(); // .gp_seq before second .gp_start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) // And ditto here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) return !time_before(j, READ_ONCE(rcu_state.gp_start) + d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) /* Don't do RCU CPU stall warnings during long sysrq printouts. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) void rcu_sysrq_start(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) if (!rcu_cpu_stall_suppress)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) rcu_cpu_stall_suppress = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) void rcu_sysrq_end(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (rcu_cpu_stall_suppress == 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) rcu_cpu_stall_suppress = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) /* Don't print RCU CPU stall warnings during a kernel panic. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) rcu_cpu_stall_suppress = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static struct notifier_block rcu_panic_block = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) .notifier_call = rcu_panic,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static int __init check_cpu_stall_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) early_initcall(check_cpu_stall_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) static void panic_on_rcu_stall(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (sysctl_panic_on_rcu_stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) panic("RCU Stall\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * rcu_cpu_stall_reset - prevent further stall warnings in current grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * Set the stall-warning timeout way off into the future, thus preventing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * any RCU CPU stall-warning messages from appearing in the current set of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * RCU grace periods.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * The caller must disable hard irqs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) void rcu_cpu_stall_reset(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) WRITE_ONCE(rcu_state.jiffies_stall, jiffies + ULONG_MAX / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) //////////////////////////////////////////////////////////////////////////////
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) //
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) // Interaction with RCU grace periods
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /* Start of new grace period, so record stall time (and forcing times). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void record_gp_stall_check_time(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) unsigned long j = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) unsigned long j1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) WRITE_ONCE(rcu_state.gp_start, j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) j1 = rcu_jiffies_till_stall_check();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) smp_mb(); // ->gp_start before ->jiffies_stall and caller's ->gp_seq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) WRITE_ONCE(rcu_state.jiffies_stall, j + j1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) rcu_state.jiffies_resched = j + j1 / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /* Zero ->ticks_this_gp and snapshot the number of RCU softirq handlers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) static void zero_cpu_stall_ticks(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) rdp->ticks_this_gp = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) WRITE_ONCE(rdp->last_fqs_resched, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * If too much time has passed in the current grace period, and if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * so configured, go kick the relevant kthreads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) static void rcu_stall_kick_kthreads(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (!READ_ONCE(rcu_kick_kthreads))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) j = READ_ONCE(rcu_state.jiffies_kick_kthreads);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (time_after(jiffies, j) && rcu_state.gp_kthread &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) (rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) WARN_ONCE(1, "Kicking %s grace-period kthread\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) rcu_state.name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) rcu_ftrace_dump(DUMP_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) wake_up_process(rcu_state.gp_kthread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * Handler for the irq_work request posted about halfway into the RCU CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * stall timeout, and used to detect excessive irq disabling. Set state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * appropriately, but just complain if there is unexpected state on entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) static void rcu_iw_handler(struct irq_work *iwp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) rdp = container_of(iwp, struct rcu_data, rcu_iw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) raw_spin_lock_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) rdp->rcu_iw_gp_seq = rnp->gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) rdp->rcu_iw_pending = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) raw_spin_unlock_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) //////////////////////////////////////////////////////////////////////////////
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) //
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) // Printing RCU CPU stall warnings
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) #ifdef CONFIG_PREEMPT_RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * Dump detailed information for all tasks blocking the current RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * grace period on the specified rcu_node structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct task_struct *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) if (!rcu_preempt_blocked_readers_cgp(rnp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) t = list_entry(rnp->gp_tasks->prev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct task_struct, rcu_node_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * We could be printing a lot while holding a spinlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * Avoid triggering hard lockup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) touch_nmi_watchdog();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) sched_show_task(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) // Communicate task state back to the RCU CPU stall warning request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) struct rcu_stall_chk_rdr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) int nesting;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) union rcu_special rs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) bool on_blkd_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * Report out the state of a not-running task that is stalling the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * current RCU grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static bool check_slow_task(struct task_struct *t, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) struct rcu_stall_chk_rdr *rscrp = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (task_curr(t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return false; // It is running, so decline to inspect it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) rscrp->nesting = t->rcu_read_lock_nesting;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) rscrp->rs = t->rcu_read_unlock_special;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) rscrp->on_blkd_list = !list_empty(&t->rcu_node_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * Scan the current list of tasks blocked within RCU read-side critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * sections, printing out the tid of each of the first few of them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) __releases(rnp->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) int ndetected = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) struct rcu_stall_chk_rdr rscr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) struct task_struct *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) struct task_struct *ts[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (!rcu_preempt_blocked_readers_cgp(rnp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) rnp->level, rnp->grplo, rnp->grphi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) t = list_entry(rnp->gp_tasks->prev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct task_struct, rcu_node_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) get_task_struct(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) ts[i++] = t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (i >= ARRAY_SIZE(ts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) while (i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) t = ts[--i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (!try_invoke_on_locked_down_task(t, check_slow_task, &rscr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) pr_cont(" P%d", t->pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) pr_cont(" P%d/%d:%c%c%c%c",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) t->pid, rscr.nesting,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) ".b"[rscr.rs.b.blocked],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) ".q"[rscr.rs.b.need_qs],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) ".e"[rscr.rs.b.exp_hint],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) ".l"[rscr.on_blkd_list]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) put_task_struct(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) ndetected++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return ndetected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #else /* #ifdef CONFIG_PREEMPT_RCU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * Because preemptible RCU does not exist, we never have to check for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * tasks blocked within RCU read-side critical sections.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * Because preemptible RCU does not exist, we never have to check for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * tasks blocked within RCU read-side critical sections.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * Dump stacks of all tasks running on stalled CPUs. First try using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * NMIs, but fall back to manual remote stack tracing on architectures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * that don't support NMI-based stack dumps. The NMI-triggered stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * traces are more accurate because they are printed by the target CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) static void rcu_dump_cpu_stacks(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) rcu_for_each_leaf_node(rnp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) for_each_leaf_node_possible_cpu(rnp, cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (!trigger_single_cpu_backtrace(cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) dump_cpu_task(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) #ifdef CONFIG_RCU_FAST_NO_HZ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) sprintf(cp, "last_accelerate: %04lx/%04lx dyntick_enabled: %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) rdp->last_accelerate & 0xffff, jiffies & 0xffff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) !!rdp->tick_nohz_enabled_snap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) *cp = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static const char * const gp_state_names[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) [RCU_GP_IDLE] = "RCU_GP_IDLE",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) [RCU_GP_WAIT_GPS] = "RCU_GP_WAIT_GPS",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) [RCU_GP_DONE_GPS] = "RCU_GP_DONE_GPS",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) [RCU_GP_ONOFF] = "RCU_GP_ONOFF",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) [RCU_GP_INIT] = "RCU_GP_INIT",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) [RCU_GP_WAIT_FQS] = "RCU_GP_WAIT_FQS",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) [RCU_GP_DOING_FQS] = "RCU_GP_DOING_FQS",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) [RCU_GP_CLEANUP] = "RCU_GP_CLEANUP",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) [RCU_GP_CLEANED] = "RCU_GP_CLEANED",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * Convert a ->gp_state value to a character string.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static const char *gp_state_getname(short gs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (gs < 0 || gs >= ARRAY_SIZE(gp_state_names))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) return "???";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return gp_state_names[gs];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) /* Is the RCU grace-period kthread being starved of CPU time? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) static bool rcu_is_gp_kthread_starving(unsigned long *jp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (jp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) *jp = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) return j > 2 * HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) * Print out diagnostic information for the specified stalled CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * If the specified CPU is aware of the current RCU grace period, then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * print the number of scheduling clock interrupts the CPU has taken
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * during the time that it has been aware. Otherwise, print the number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * of RCU grace periods that this CPU is ignorant of, for example, "1"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * if the CPU was aware of the previous grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static void print_cpu_stall_info(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) unsigned long delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) bool falsepositive;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) char fast_no_hz[72];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) char *ticks_title;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) unsigned long ticks_value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * We could be printing a lot while holding a spinlock. Avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * triggering hard lockup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) touch_nmi_watchdog();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) ticks_value = rcu_seq_ctr(rcu_state.gp_seq - rdp->gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) if (ticks_value) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) ticks_title = "GPs behind";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) ticks_title = "ticks this GP";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) ticks_value = rdp->ticks_this_gp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) falsepositive = rcu_is_gp_kthread_starving(NULL) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) pr_err("\t%d-%c%c%c%c: (%lu %s) idle=%03x/%ld/%#lx softirq=%u/%u fqs=%ld %s%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) "O."[!!cpu_online(cpu)],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) !IS_ENABLED(CONFIG_IRQ_WORK) ? '?' :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) "!."[!delta],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) ticks_value, ticks_title,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) rcu_dynticks_snap(rdp) & 0xfff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) rdp->dynticks_nesting, rdp->dynticks_nmi_nesting,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) fast_no_hz,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) falsepositive ? " (false positive?)" : "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /* Complain about starvation of grace-period kthread. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) static void rcu_check_gp_kthread_starvation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) struct task_struct *gpk = rcu_state.gp_kthread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (rcu_is_gp_kthread_starving(&j)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) rcu_state.name, j,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) (long)rcu_seq_current(&rcu_state.gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) data_race(rcu_state.gp_flags),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) gp_state_getname(rcu_state.gp_state), rcu_state.gp_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) gpk ? gpk->state : ~0, gpk ? task_cpu(gpk) : -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) if (gpk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) pr_err("\tUnless %s kthread gets sufficient CPU time, OOM is now expected behavior.\n", rcu_state.name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) pr_err("RCU grace-period kthread stack dump:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) sched_show_task(gpk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) wake_up_process(gpk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) unsigned long gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) int ndetected = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) long totqlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) /* Kick and suppress, if so configured. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) rcu_stall_kick_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (rcu_stall_is_suppressed())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * OK, time to rat on our buddy...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * See Documentation/RCU/stallwarn.rst for info on how to debug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * RCU CPU stall warnings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) trace_rcu_stall_warning(rcu_state.name, TPS("StallDetected"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) rcu_for_each_leaf_node(rnp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) if (rnp->qsmask != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) for_each_leaf_node_possible_cpu(rnp, cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) print_cpu_stall_info(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) ndetected++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) ndetected += rcu_print_task_stall(rnp, flags); // Releases rnp->lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) totqlen += rcu_get_n_cbs_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) pr_cont("\t(detected by %d, t=%ld jiffies, g=%ld, q=%lu)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) smp_processor_id(), (long)(jiffies - gps),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) (long)rcu_seq_current(&rcu_state.gp_seq), totqlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (ndetected) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) rcu_dump_cpu_stacks();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /* Complain about tasks blocking the grace period. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) rcu_for_each_leaf_node(rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) rcu_print_detail_task_stall_rnp(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (rcu_seq_current(&rcu_state.gp_seq) != gp_seq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) pr_err("INFO: Stall ended before state dump start\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) j = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) gpa = data_race(rcu_state.gp_activity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) rcu_state.name, j - gpa, j, gpa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) data_race(jiffies_till_next_fqs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) rcu_get_root()->qsmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) /* Rewrite if needed in case of slow consoles. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) WRITE_ONCE(rcu_state.jiffies_stall,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) rcu_check_gp_kthread_starvation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) atomic_notifier_call_chain(&rcu_stall_notifier_list, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) panic_on_rcu_stall();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) rcu_force_quiescent_state(); /* Kick them all. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) static void print_cpu_stall(unsigned long gps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) struct rcu_node *rnp = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) long totqlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) /* Kick and suppress, if so configured. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) rcu_stall_kick_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (rcu_stall_is_suppressed())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * OK, time to rat on ourselves...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * See Documentation/RCU/stallwarn.rst for info on how to debug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) * RCU CPU stall warnings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) trace_rcu_stall_warning(rcu_state.name, TPS("SelfDetected"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) raw_spin_lock_irqsave_rcu_node(rdp->mynode, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) print_cpu_stall_info(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) raw_spin_unlock_irqrestore_rcu_node(rdp->mynode, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) totqlen += rcu_get_n_cbs_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) pr_cont("\t(t=%lu jiffies g=%ld q=%lu)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) jiffies - gps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) (long)rcu_seq_current(&rcu_state.gp_seq), totqlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) rcu_check_gp_kthread_starvation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) rcu_dump_cpu_stacks();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /* Rewrite if needed in case of slow consoles. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) WRITE_ONCE(rcu_state.jiffies_stall,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) jiffies + 3 * rcu_jiffies_till_stall_check() + 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) panic_on_rcu_stall();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * Attempt to revive the RCU machinery by forcing a context switch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * A context switch would normally allow the RCU state machine to make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * progress and it could be we're stuck in kernel space without context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * switches for an entirely unreasonable amount of time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) set_tsk_need_resched(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) set_preempt_need_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) static void check_cpu_stall(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) unsigned long gs1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) unsigned long gs2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) unsigned long gps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) unsigned long jn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) unsigned long js;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) if ((rcu_stall_is_suppressed() && !READ_ONCE(rcu_kick_kthreads)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) !rcu_gp_in_progress())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) rcu_stall_kick_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) j = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * Lots of memory barriers to reject false positives.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * The idea is to pick up rcu_state.gp_seq, then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * rcu_state.jiffies_stall, then rcu_state.gp_start, and finally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * another copy of rcu_state.gp_seq. These values are updated in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * the opposite order with memory barriers (or equivalent) during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * grace-period initialization and cleanup. Now, a false positive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) * can occur if we get an new value of rcu_state.gp_start and a old
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * value of rcu_state.jiffies_stall. But given the memory barriers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * the only way that this can happen is if one grace period ends
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * and another starts between these two fetches. This is detected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * by comparing the second fetch of rcu_state.gp_seq with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * previous fetch from rcu_state.gp_seq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * Given this check, comparisons of jiffies, rcu_state.jiffies_stall,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * and rcu_state.gp_start suffice to forestall false positives.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) gs1 = READ_ONCE(rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) smp_rmb(); /* Pick up ->gp_seq first... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) js = READ_ONCE(rcu_state.jiffies_stall);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) smp_rmb(); /* ...then ->jiffies_stall before the rest... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) gps = READ_ONCE(rcu_state.gp_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) smp_rmb(); /* ...and finally ->gp_start before ->gp_seq again. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) gs2 = READ_ONCE(rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) if (gs1 != gs2 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) ULONG_CMP_LT(j, js) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) ULONG_CMP_GE(gps, js))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) return; /* No stall or GP completed since entering function. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) jn = jiffies + 3 * rcu_jiffies_till_stall_check() + 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (rcu_gp_in_progress() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) (READ_ONCE(rnp->qsmask) & rdp->grpmask) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) * If a virtual machine is stopped by the host it can look to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * the watchdog like an RCU stall. Check to see if the host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * stopped the vm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (kvm_check_and_clear_guest_paused())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) /* We haven't checked in, so go dump stack. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) print_cpu_stall(gps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) if (READ_ONCE(rcu_cpu_stall_ftrace_dump))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) rcu_ftrace_dump(DUMP_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) } else if (rcu_gp_in_progress() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * If a virtual machine is stopped by the host it can look to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * the watchdog like an RCU stall. Check to see if the host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * stopped the vm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) if (kvm_check_and_clear_guest_paused())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) /* They had a few time units to dump stack, so complain. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) print_other_cpu_stall(gs2, gps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (READ_ONCE(rcu_cpu_stall_ftrace_dump))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) rcu_ftrace_dump(DUMP_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) //////////////////////////////////////////////////////////////////////////////
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) //
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) // RCU forward-progress mechanisms, including of callback invocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) * Show the state of the grace-period kthreads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) void show_rcu_gp_kthreads(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) unsigned long cbs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) unsigned long ja;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) unsigned long jr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) unsigned long jw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) j = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) ja = j - data_race(rcu_state.gp_activity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) jr = j - data_race(rcu_state.gp_req_activity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) jw = j - data_race(rcu_state.gp_wake_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) rcu_state.name, gp_state_getname(rcu_state.gp_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) rcu_state.gp_state, t ? t->state : 0x1ffffL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) ja, jr, jw, (long)data_race(rcu_state.gp_wake_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) (long)data_race(rcu_state.gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) (long)data_race(rcu_get_root()->gp_seq_needed),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) data_race(rcu_state.gp_flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) rcu_for_each_node_breadth_first(rnp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) READ_ONCE(rnp->gp_seq_needed)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) rnp->grplo, rnp->grphi, (long)data_race(rnp->gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) (long)data_race(rnp->gp_seq_needed));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) if (!rcu_is_leaf_node(rnp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) for_each_leaf_node_possible_cpu(rnp, cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (READ_ONCE(rdp->gpwrap) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) READ_ONCE(rdp->gp_seq_needed)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) pr_info("\tcpu %d ->gp_seq_needed %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) cpu, (long)data_race(rdp->gp_seq_needed));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) cbs += data_race(rdp->n_cbs_invoked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) if (rcu_segcblist_is_offloaded(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) show_rcu_nocb_state(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) pr_info("RCU callbacks invoked since boot: %lu\n", cbs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) show_rcu_tasks_gp_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * This function checks for grace-period requests that fail to motivate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * RCU to come out of its idle mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) const unsigned long gpssdelay)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) struct rcu_node *rnp_root = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) static atomic_t warned = ATOMIC_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) if (!IS_ENABLED(CONFIG_PROVE_RCU) || rcu_gp_in_progress() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) READ_ONCE(rnp_root->gp_seq_needed)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) !smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) j = jiffies; /* Expensive access, and in common case don't get here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) atomic_read(&warned))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) j = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (rcu_gp_in_progress() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) READ_ONCE(rnp_root->gp_seq_needed)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) atomic_read(&warned)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) /* Hold onto the leaf lock to make others see warned==1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) if (rnp_root != rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) j = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) if (rcu_gp_in_progress() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) READ_ONCE(rnp_root->gp_seq_needed)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) atomic_xchg(&warned, 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (rnp_root != rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) /* irqs remain disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) raw_spin_unlock_rcu_node(rnp_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) if (rnp_root != rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) raw_spin_unlock_rcu_node(rnp_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) show_rcu_gp_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * Do a forward-progress check for rcutorture. This is normally invoked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) * due to an OOM event. The argument "j" gives the time period during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) * which rcutorture would like progress to have been made.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) void rcu_fwd_progress_check(unsigned long j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) unsigned long cbs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) unsigned long max_cbs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) int max_cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) if (rcu_gp_in_progress()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) pr_info("%s: GP age %lu jiffies\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) __func__, jiffies - rcu_state.gp_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) show_rcu_gp_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) pr_info("%s: Last GP end %lu jiffies ago\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) __func__, jiffies - rcu_state.gp_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) rcu_check_gp_start_stall(rdp->mynode, rdp, j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) cbs = rcu_get_n_cbs_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) if (!cbs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (max_cpu < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) pr_info("%s: callbacks", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) pr_cont(" %d: %lu", cpu, cbs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) if (cbs <= max_cbs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) max_cbs = cbs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) max_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (max_cpu >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) EXPORT_SYMBOL_GPL(rcu_fwd_progress_check);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) /* Commandeer a sysrq key to dump RCU's tree. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) static bool sysrq_rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) module_param(sysrq_rcu, bool, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) /* Dump grace-period-request information due to commandeered sysrq. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) static void sysrq_show_rcu(int key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) show_rcu_gp_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) static const struct sysrq_key_op sysrq_rcudump_op = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) .handler = sysrq_show_rcu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) .help_msg = "show-rcu(y)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) .action_msg = "Show RCU tree",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) .enable_mask = SYSRQ_ENABLE_DUMP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) static int __init rcu_sysrq_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) if (sysrq_rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) return register_sysrq_key('y', &sysrq_rcudump_op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) early_initcall(rcu_sysrq_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags