^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Read-Copy Update mechanism for mutual exclusion (tree-based version)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright IBM Corporation, 2008
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Authors: Dipankar Sarma <dipankar@in.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Manfred Spraul <manfred@colorfullife.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Paul E. McKenney <paulmck@linux.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * For detailed explanation of Read-Copy Update mechanism see -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Documentation/RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define pr_fmt(fmt) "rcu: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/rcupdate_wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/sched/debug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/completion.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/moduleparam.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/time.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/kernel_stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <uapi/linux/sched/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/prefetch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/random.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/trace_events.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/suspend.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/tick.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/sysrq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/oom.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <linux/smpboot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <linux/sched/isolation.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <linux/sched/clock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #include <linux/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include "../time/tick-internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include "tree.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include "rcu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #ifdef MODULE_PARAM_PREFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #undef MODULE_PARAM_PREFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define MODULE_PARAM_PREFIX "rcutree."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* Data structures. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * Steal a bit from the bottom of ->dynticks for idle entry/exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * control. Initially this is for TLB flushing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #define RCU_DYNTICK_CTRL_MASK 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) .dynticks_nesting = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static struct rcu_state rcu_state = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) .level = { &rcu_state.node[0] },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) .gp_state = RCU_GP_IDLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) .name = RCU_NAME,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) .abbr = RCU_ABBR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /* Dump rcu_node combining tree at boot to verify correct setup. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static bool dump_tree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) module_param(dump_tree, bool, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) /* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static bool use_softirq = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) module_param(use_softirq, bool, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /* Control rcu_node-tree auto-balancing at boot time. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static bool rcu_fanout_exact;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) module_param(rcu_fanout_exact, bool, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static int rcu_fanout_leaf = RCU_FANOUT_LEAF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) module_param(rcu_fanout_leaf, int, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /* Number of rcu_nodes at specified level. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) int num_rcu_lvl[] = NUM_RCU_LVL_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * The rcu_scheduler_active variable is initialized to the value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * RCU can assume that there is but one task, allowing RCU to (for example)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * optimize synchronize_rcu() to a simple barrier(). When this variable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * to detect real grace periods. This variable is also used to suppress
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * boot-time false positives from lockdep-RCU error checking. Finally, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * is fully initialized, including all of its kthreads having been spawned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) int rcu_scheduler_active __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) EXPORT_SYMBOL_GPL(rcu_scheduler_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * The rcu_scheduler_fully_active variable transitions from zero to one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * during the early_initcall() processing, which is after the scheduler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * is capable of creating new tasks. So RCU processing (for example,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * creating tasks for RCU priority boosting) must be delayed until after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * rcu_scheduler_fully_active transitions from zero to one. We also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * currently delay invocation of any RCU callbacks until after this point.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * It might later prove better for people registering RCU callbacks during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * early boot to take responsibility for these callbacks, but one step at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * a time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static int rcu_scheduler_fully_active __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) unsigned long gps, unsigned long flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static void invoke_rcu_core(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static void rcu_report_exp_rdp(struct rcu_data *rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) static void sync_sched_exp_online_cleanup(int cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /* rcuc/rcub kthread realtime priority */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) module_param(kthread_prio, int, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* Delay in jiffies for grace-period initialization delays, debug only. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static int gp_preinit_delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) module_param(gp_preinit_delay, int, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static int gp_init_delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) module_param(gp_init_delay, int, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static int gp_cleanup_delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) module_param(gp_cleanup_delay, int, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) // Add delay to rcu_read_unlock() for strict grace periods.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) static int rcu_unlock_delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) #ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) module_param(rcu_unlock_delay, int, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * This rcu parameter is runtime-read-only. It reflects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * a minimum allowed number of objects which can be cached
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * per-CPU. Object size is equal to one page. This value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * can be changed at boot time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) static int rcu_min_cached_objs = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) module_param(rcu_min_cached_objs, int, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) /* Retrieve RCU kthreads priority for rcutorture */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) int rcu_get_gp_kthreads_prio(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) return kthread_prio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * Number of grace periods between delays, normalized by the duration of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * the delay. The longer the delay, the more the grace periods between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * each delay. The reason for this normalization is that it means that,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * for non-zero delays, the overall slowdown of grace periods is constant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * regardless of the duration of the delay. This arrangement balances
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * the need for long delays to increase some race probabilities with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * need for fast grace periods to increase other race probabilities.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) #define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * Compute the mask of online CPUs for the specified rcu_node structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * This will not be stable unless the rcu_node structure's ->lock is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * held, but the bit corresponding to the current CPU will be stable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * in most contexts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return READ_ONCE(rnp->qsmaskinitnext);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * Return true if an RCU grace period is in progress. The READ_ONCE()s
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * permit this function to be invoked without holding the root rcu_node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * structure's ->lock, but of course results can be subject to change.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static int rcu_gp_in_progress(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * Return the number of callbacks queued on the specified CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * Handles both the nocbs and normal cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static long rcu_get_n_cbs_cpu(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (rcu_segcblist_is_enabled(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) return rcu_segcblist_n_cbs(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) void rcu_softirq_qs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) rcu_preempt_deferred_qs(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * Record entry into an extended quiescent state. This is only to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * called when not already in an extended quiescent state, that is,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * RCU is watching prior to the call to this function and is no longer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * watching upon return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) static noinstr void rcu_dynticks_eqs_enter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) int seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * CPUs seeing atomic_add_return() must see prior RCU read-side
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * critical sections, and we also must force ordering with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * next idle sojourn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) rcu_dynticks_task_trace_enter(); // Before ->dynticks update!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) // RCU is no longer watching. Better be in extended quiescent state!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) (seq & RCU_DYNTICK_CTRL_CTR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /* Better not have special action (TLB flush) pending! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) (seq & RCU_DYNTICK_CTRL_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * Record exit from an extended quiescent state. This is only to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * called from an extended quiescent state, that is, RCU is not watching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * prior to the call to this function and is watching upon return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) static noinstr void rcu_dynticks_eqs_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) int seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * CPUs seeing atomic_add_return() must see prior idle sojourns,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * and we also must force ordering with the next RCU read-side
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) // RCU is now watching. Better not be in an extended quiescent state!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) rcu_dynticks_task_trace_exit(); // After ->dynticks update!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) !(seq & RCU_DYNTICK_CTRL_CTR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) if (seq & RCU_DYNTICK_CTRL_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) smp_mb__after_atomic(); /* _exit after clearing mask. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * Reset the current CPU's ->dynticks counter to indicate that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * newly onlined CPU is no longer in an extended quiescent state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * This will either leave the counter unchanged, or increment it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * to the next non-quiescent value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * The non-atomic test/increment sequence works because the upper bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * of the ->dynticks counter are manipulated only by the corresponding CPU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * or when the corresponding CPU is offline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static void rcu_dynticks_eqs_online(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * Is the current CPU in an extended quiescent state?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * No ordering, as we are sampling CPU-local information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * Snapshot the ->dynticks counter with full ordering so as to allow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * stable comparison of this counter with past and future snapshots.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) static int rcu_dynticks_snap(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) int snap = atomic_add_return(0, &rdp->dynticks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return snap & ~RCU_DYNTICK_CTRL_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * Return true if the snapshot returned from rcu_dynticks_snap()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * indicates that RCU is in an extended quiescent state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static bool rcu_dynticks_in_eqs(int snap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) return !(snap & RCU_DYNTICK_CTRL_CTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * Return true if the CPU corresponding to the specified rcu_data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * structure has spent some time in an extended quiescent state since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * rcu_dynticks_snap() returned the specified snapshot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return snap != rcu_dynticks_snap(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * Return true if the referenced integer is zero while the specified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * CPU remains within a single extended quiescent state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) bool rcu_dynticks_zero_in_eqs(int cpu, int *vp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) int snap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) // If not quiescent, force back to earlier extended quiescent state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) snap = atomic_read(&rdp->dynticks) & ~(RCU_DYNTICK_CTRL_MASK |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) RCU_DYNTICK_CTRL_CTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) smp_rmb(); // Order ->dynticks and *vp reads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (READ_ONCE(*vp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) return false; // Non-zero, so report failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) smp_rmb(); // Order *vp read and ->dynticks re-read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) // If still in the same extended quiescent state, we are good!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) return snap == (atomic_read(&rdp->dynticks) & ~RCU_DYNTICK_CTRL_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * Set the special (bottom) bit of the specified CPU so that it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * will take special action (such as flushing its TLB) on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * next exit from an extended quiescent state. Returns true if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * the bit was successfully set, or false if the CPU was not in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * an extended quiescent state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) bool rcu_eqs_special_set(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) int old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) int new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) int new_old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) new_old = atomic_read(&rdp->dynticks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) old = new_old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (old & RCU_DYNTICK_CTRL_CTR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) new = old | RCU_DYNTICK_CTRL_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) new_old = atomic_cmpxchg(&rdp->dynticks, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) } while (new_old != old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * Let the RCU core know that this CPU has gone through the scheduler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * which is a quiescent state. This is called when the need for a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * quiescent state is urgent, so we burn an atomic operation and full
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * memory barriers to let the RCU core know about it, regardless of what
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * this CPU might (or might not) do in the near future.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) * We inform the RCU core by emulating a zero-duration dyntick-idle period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * The caller must have disabled interrupts and must not be idle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) notrace void rcu_momentary_dyntick_idle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) int special;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) raw_cpu_write(rcu_data.rcu_need_heavy_qs, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) &this_cpu_ptr(&rcu_data)->dynticks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /* It is illegal to call this from idle state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) rcu_preempt_deferred_qs(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * If the current CPU is idle and running at a first-level (not nested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * interrupt, or directly, from idle, return true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * The caller must have at least disabled IRQs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) static int rcu_is_cpu_rrupt_from_idle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) long nesting;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) * Usually called from the tick; but also used from smp_function_call()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * for expedited grace periods. This latter can result in running from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * the idle task, instead of an actual IPI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /* Check for counter underflows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) "RCU dynticks_nesting counter underflow!");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) "RCU dynticks_nmi_nesting counter underflow/zero!");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /* Are we at first interrupt nesting level? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (nesting > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * If we're not in an interrupt, we must be in the idle task!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) WARN_ON_ONCE(!nesting && !is_idle_task(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /* Does CPU appear to be idle from an RCU standpoint? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return __this_cpu_read(rcu_data.dynticks_nesting) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) #define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) // Maximum callbacks per rcu_do_batch ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) #define DEFAULT_MAX_RCU_BLIMIT 10000 // ... even during callback flood.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) static long blimit = DEFAULT_RCU_BLIMIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) #define DEFAULT_RCU_QHIMARK 10000 // If this many pending, ignore blimit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) static long qhimark = DEFAULT_RCU_QHIMARK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) #define DEFAULT_RCU_QLOMARK 100 // Once only this many pending, use blimit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) static long qlowmark = DEFAULT_RCU_QLOMARK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) #define DEFAULT_RCU_QOVLD_MULT 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) #define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) static long qovld = DEFAULT_RCU_QOVLD; // If this many pending, hammer QS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) static long qovld_calc = -1; // No pre-initialization lock acquisitions!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) module_param(blimit, long, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) module_param(qhimark, long, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) module_param(qlowmark, long, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) module_param(qovld, long, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) static ulong jiffies_till_first_fqs = IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 0 : ULONG_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) static ulong jiffies_till_next_fqs = ULONG_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) static bool rcu_kick_kthreads;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) static int rcu_divisor = 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) module_param(rcu_divisor, int, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /* Force an exit from rcu_do_batch() after 3 milliseconds. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) static long rcu_resched_ns = 3 * NSEC_PER_MSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) module_param(rcu_resched_ns, long, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * How long the grace period must be before we start recruiting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * quiescent-state help from rcu_note_context_switch().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) static ulong jiffies_till_sched_qs = ULONG_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) module_param(jiffies_till_sched_qs, ulong, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static ulong jiffies_to_sched_qs; /* See adjust_jiffies_till_sched_qs(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * Make sure that we give the grace-period kthread time to detect any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * idle CPUs before taking active measures to force quiescent states.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * However, don't go below 100 milliseconds, adjusted upwards for really
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * large systems.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) static void adjust_jiffies_till_sched_qs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) /* If jiffies_till_sched_qs was specified, respect the request. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (jiffies_till_sched_qs != ULONG_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) /* Otherwise, set to third fqs scan, but bound below on large system. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) j = READ_ONCE(jiffies_till_first_fqs) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) 2 * READ_ONCE(jiffies_till_next_fqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) WRITE_ONCE(jiffies_to_sched_qs, j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) static int param_set_first_fqs_jiffies(const char *val, const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) ulong j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) int ret = kstrtoul(val, 0, &j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) adjust_jiffies_till_sched_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) ulong j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) int ret = kstrtoul(val, 0, &j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) adjust_jiffies_till_sched_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) static struct kernel_param_ops first_fqs_jiffies_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) .set = param_set_first_fqs_jiffies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) .get = param_get_ulong,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) static struct kernel_param_ops next_fqs_jiffies_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) .set = param_set_next_fqs_jiffies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) .get = param_get_ulong,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) module_param_cb(jiffies_till_first_fqs, &first_fqs_jiffies_ops, &jiffies_till_first_fqs, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) module_param_cb(jiffies_till_next_fqs, &next_fqs_jiffies_ops, &jiffies_till_next_fqs, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) module_param(rcu_kick_kthreads, bool, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) static int rcu_pending(int user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * Return the number of RCU GPs completed thus far for debug & stats.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) unsigned long rcu_get_gp_seq(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) return READ_ONCE(rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * Return the number of RCU expedited batches completed thus far for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * debug & stats. Odd numbers mean that a batch is in progress, even
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) * numbers mean idle. The value returned will thus be roughly double
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) * the cumulative batches since boot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) unsigned long rcu_exp_batches_completed(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) return rcu_state.expedited_sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) EXPORT_SYMBOL_GPL(rcu_exp_batches_completed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * Return the root node of the rcu_state structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) static struct rcu_node *rcu_get_root(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) return &rcu_state.node[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) * Send along grace-period-related data for rcutorture diagnostics.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) unsigned long *gp_seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) switch (test_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) case RCU_FLAVOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) *flags = READ_ONCE(rcu_state.gp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * Enter an RCU extended quiescent state, which can be either the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) * idle loop or adaptive-tickless usermode execution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * We crowbar the ->dynticks_nmi_nesting field to zero to allow for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * the possibility of usermode upcalls having messed up our count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) * of interrupt nesting level during the prior busy period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) static noinstr void rcu_eqs_enter(bool user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) WRITE_ONCE(rdp->dynticks_nmi_nesting, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) rdp->dynticks_nesting == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (rdp->dynticks_nesting != 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) // RCU will still be watching, so just do accounting and leave.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) rdp->dynticks_nesting--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) rcu_prepare_for_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) rcu_preempt_deferred_qs(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) // instrumentation for the noinstr rcu_dynticks_eqs_enter()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) instrumentation_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) // RCU is watching here ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) rcu_dynticks_eqs_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) // ... but is no longer watching here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) rcu_dynticks_task_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) * rcu_idle_enter - inform RCU that current CPU is entering idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * Enter idle mode, in other words, -leave- the mode in which RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * read-side critical sections can occur. (Though RCU read-side
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) * critical sections can occur in irq handlers in idle, a possibility
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) * handled by irq_enter() and irq_exit().)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * If you add or remove a call to rcu_idle_enter(), be sure to test with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) void rcu_idle_enter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) rcu_eqs_enter(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) EXPORT_SYMBOL_GPL(rcu_idle_enter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) #ifdef CONFIG_NO_HZ_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * rcu_user_enter - inform RCU that we are resuming userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) * Enter RCU idle mode right before resuming userspace. No use of RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * is permitted between this call and rcu_user_exit(). This way the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * CPU doesn't need to maintain the tick for RCU maintenance purposes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * when the CPU runs in userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * If you add or remove a call to rcu_user_enter(), be sure to test with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) * CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) noinstr void rcu_user_enter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) do_nocb_deferred_wakeup(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) instrumentation_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) rcu_eqs_enter(true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) #endif /* CONFIG_NO_HZ_FULL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * rcu_nmi_exit - inform RCU of exit from NMI context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * If we are returning from the outermost NMI handler that interrupted an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * to let the RCU grace-period handling know that the CPU is back to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * being RCU-idle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * If you add or remove a call to rcu_nmi_exit(), be sure to test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * with CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) noinstr void rcu_nmi_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * (We are exiting an NMI handler, so RCU better be paying attention
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) * to us!)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * If the nesting level is not 1, the CPU wasn't RCU-idle, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) * leave it in non-RCU-idle state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (rdp->dynticks_nmi_nesting != 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) atomic_read(&rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) rdp->dynticks_nmi_nesting - 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) instrumentation_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) if (!in_nmi())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) rcu_prepare_for_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) // instrumentation for the noinstr rcu_dynticks_eqs_enter()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) instrumentation_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) // RCU is watching here ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) rcu_dynticks_eqs_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) // ... but is no longer watching here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) if (!in_nmi())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) rcu_dynticks_task_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * Exit from an interrupt handler, which might possibly result in entering
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * idle mode, in other words, leaving the mode in which read-side critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * sections can occur. The caller must have disabled interrupts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * This code assumes that the idle loop never does anything that might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * result in unbalanced calls to irq_enter() and irq_exit(). If your
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * architecture's idle loop violates this assumption, RCU will give you what
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * you deserve, good and hard. But very infrequently and irreproducibly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * Use things like work queues to work around this limitation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * You have been warned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * If you add or remove a call to rcu_irq_exit(), be sure to test with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) void noinstr rcu_irq_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) rcu_nmi_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * rcu_irq_exit_preempt - Inform RCU that current CPU is exiting irq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * towards in kernel preemption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) * Same as rcu_irq_exit() but has a sanity check that scheduling is safe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * from RCU point of view. Invoked from return from interrupt before kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * preemption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) void rcu_irq_exit_preempt(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) rcu_nmi_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) "RCU dynticks_nesting counter underflow/zero!");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) DYNTICK_IRQ_NONIDLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) "Bad RCU dynticks_nmi_nesting counter\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) "RCU in extended quiescent state!");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) #ifdef CONFIG_PROVE_RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * rcu_irq_exit_check_preempt - Validate that scheduling is possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) void rcu_irq_exit_check_preempt(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) "RCU dynticks_nesting counter underflow/zero!");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) DYNTICK_IRQ_NONIDLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) "Bad RCU dynticks_nmi_nesting counter\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) "RCU in extended quiescent state!");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) #endif /* #ifdef CONFIG_PROVE_RCU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) * Wrapper for rcu_irq_exit() where interrupts are enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) * If you add or remove a call to rcu_irq_exit_irqson(), be sure to test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) * with CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) void rcu_irq_exit_irqson(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) rcu_irq_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * Exit an RCU extended quiescent state, which can be either the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) * idle loop or adaptive-tickless usermode execution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * allow for the possibility of usermode upcalls messing up our count of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * interrupt nesting level during the busy period that is just now starting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) static void noinstr rcu_eqs_exit(bool user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) long oldval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) oldval = rdp->dynticks_nesting;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) if (oldval) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) // RCU was already watching, so just do accounting and leave.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) rdp->dynticks_nesting++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) rcu_dynticks_task_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) // RCU is not watching here ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) rcu_dynticks_eqs_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) // ... but is watching here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) // instrumentation for the noinstr rcu_dynticks_eqs_exit()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) rcu_cleanup_after_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) WRITE_ONCE(rdp->dynticks_nesting, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) WARN_ON_ONCE(rdp->dynticks_nmi_nesting);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) instrumentation_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * rcu_idle_exit - inform RCU that current CPU is leaving idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * Exit idle mode, in other words, -enter- the mode in which RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * read-side critical sections can occur.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * If you add or remove a call to rcu_idle_exit(), be sure to test with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) void rcu_idle_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) rcu_eqs_exit(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) EXPORT_SYMBOL_GPL(rcu_idle_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) #ifdef CONFIG_NO_HZ_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * rcu_user_exit - inform RCU that we are exiting userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * Exit RCU idle mode while entering the kernel because it can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * run a RCU read side critical section anytime.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * If you add or remove a call to rcu_user_exit(), be sure to test with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) void noinstr rcu_user_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) rcu_eqs_exit(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) * __rcu_irq_enter_check_tick - Enable scheduler tick on CPU if RCU needs it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) * The scheduler tick is not normally enabled when CPUs enter the kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * from nohz_full userspace execution. After all, nohz_full userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * execution is an RCU quiescent state and the time executing in the kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) * is quite short. Except of course when it isn't. And it is not hard to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) * cause a large system to spend tens of seconds or even minutes looping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * in the kernel, which can cause a number of problems, include RCU CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * stall warnings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * Therefore, if a nohz_full CPU fails to report a quiescent state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) * in a timely manner, the RCU grace-period kthread sets that CPU's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) * ->rcu_urgent_qs flag with the expectation that the next interrupt or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) * exception will invoke this function, which will turn on the scheduler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) * tick, which will enable RCU to detect that CPU's quiescent states,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) * for example, due to cond_resched() calls in CONFIG_PREEMPT=n kernels.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) * The tick will be disabled once a quiescent state is reported for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) * this CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) * Of course, in carefully tuned systems, there might never be an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * interrupt or exception. In that case, the RCU grace-period kthread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * will eventually cause one to happen. However, in less carefully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) * controlled environments, this function allows RCU to get what it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) * needs without creating otherwise useless interruptions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) void __rcu_irq_enter_check_tick(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) // If we're here from NMI there's nothing to do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) if (in_nmi())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) "Illegal rcu_irq_enter_check_tick() from extended quiescent state");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) if (!tick_nohz_full_cpu(rdp->cpu) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) !READ_ONCE(rdp->rcu_urgent_qs) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) READ_ONCE(rdp->rcu_forced_tick)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) // RCU doesn't need nohz_full help from this CPU, or it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) // already getting that help.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) // We get here only when not in an extended quiescent state and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) // from interrupts (as opposed to NMIs). Therefore, (1) RCU is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) // already watching and (2) The fact that we are in an interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) // handler and that the rcu_node lock is an irq-disabled lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) // prevents self-deadlock. So we can safely recheck under the lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) // Note that the nohz_full state currently cannot change.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) raw_spin_lock_rcu_node(rdp->mynode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) // A nohz_full CPU is in the kernel and RCU needs a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) // quiescent state. Turn on the tick!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) WRITE_ONCE(rdp->rcu_forced_tick, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) raw_spin_unlock_rcu_node(rdp->mynode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) #endif /* CONFIG_NO_HZ_FULL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) * rcu_nmi_enter - inform RCU of entry to NMI context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) * that the CPU is active. This implementation permits nested NMIs, as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) * long as the nesting level does not overflow an int. (You will probably
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) * run out of stack space first.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) * If you add or remove a call to rcu_nmi_enter(), be sure to test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) * with CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) noinstr void rcu_nmi_enter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) long incby = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) /* Complain about underflow. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) * If idle from RCU viewpoint, atomically increment ->dynticks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) * to mark non-idle and increment ->dynticks_nmi_nesting by one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) * Otherwise, increment ->dynticks_nmi_nesting by two. This means
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) * to be in the outermost NMI handler that interrupted an RCU-idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) * period (observation due to Andy Lutomirski).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) if (rcu_dynticks_curr_cpu_in_eqs()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) if (!in_nmi())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) rcu_dynticks_task_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) // RCU is not watching here ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) rcu_dynticks_eqs_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) // ... but is watching here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) rcu_cleanup_after_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) instrumentation_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) // instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) // instrumentation for the noinstr rcu_dynticks_eqs_exit()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) incby = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) } else if (!in_nmi()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) rcu_irq_enter_check_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) rdp->dynticks_nmi_nesting,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) instrumentation_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) rdp->dynticks_nmi_nesting + incby);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) * Enter an interrupt handler, which might possibly result in exiting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) * idle mode, in other words, entering the mode in which read-side critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * sections can occur. The caller must have disabled interrupts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * Note that the Linux kernel is fully capable of entering an interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) * handler that it never exits, for example when doing upcalls to user mode!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) * This code assumes that the idle loop never does upcalls to user mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) * If your architecture's idle loop does do upcalls to user mode (or does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) * anything else that results in unbalanced calls to the irq_enter() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) * irq_exit() functions), RCU will give you what you deserve, good and hard.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) * But very infrequently and irreproducibly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) * Use things like work queues to work around this limitation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) * You have been warned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) * If you add or remove a call to rcu_irq_enter(), be sure to test with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) * CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) noinstr void rcu_irq_enter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) rcu_nmi_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) * Wrapper for rcu_irq_enter() where interrupts are enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) * If you add or remove a call to rcu_irq_enter_irqson(), be sure to test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) * with CONFIG_RCU_EQS_DEBUG=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) void rcu_irq_enter_irqson(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) rcu_irq_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) * If any sort of urgency was applied to the current CPU (for example,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) * to get to a quiescent state, disable it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) raw_lockdep_assert_held_rcu_node(rdp->mynode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) WRITE_ONCE(rdp->rcu_urgent_qs, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) WRITE_ONCE(rdp->rcu_need_heavy_qs, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) WRITE_ONCE(rdp->rcu_forced_tick, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) * rcu_is_watching - see if RCU thinks that the current CPU is not idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) * Return true if RCU is watching the running CPU, which means that this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * CPU can safely enter RCU read-side critical sections. In other words,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) * if the current CPU is not in its idle loop or is in an interrupt or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * NMI handler, return true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) * Make notrace because it can be called by the internal functions of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) * ftrace, and making this notrace removes unnecessary recursion calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) notrace bool rcu_is_watching(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) preempt_disable_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) ret = !rcu_dynticks_curr_cpu_in_eqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) preempt_enable_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) EXPORT_SYMBOL_GPL(rcu_is_watching);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) * If a holdout task is actually running, request an urgent quiescent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) * state from its CPU. This is unsynchronized, so migrations can cause
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) * the request to go to the wrong CPU. Which is OK, all that will happen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) * is that the CPU's next context switch will be a bit slower and next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) * time around this task will generate another request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) void rcu_request_urgent_qs_task(struct task_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) cpu = task_cpu(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) if (!task_curr(t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) return; /* This task is not running on that CPU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) * Is the current CPU online as far as RCU is concerned?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) * Disable preemption to avoid false positives that could otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) * happen due to the current CPU number being sampled, this task being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) * preempted, its old CPU being taken offline, resuming on some other CPU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) * then determining that its old CPU is now offline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) * Disable checking if in an NMI handler because we cannot safely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) * report errors from NMI handlers anyway. In addition, it is OK to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) * RCU on an offline processor during initial boot, hence the check for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) * rcu_scheduler_fully_active.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) bool rcu_lockdep_current_cpu_online(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) if (in_nmi() || !rcu_scheduler_fully_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) preempt_disable_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) if (rdp->grpmask & rcu_rnp_online_cpus(rnp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) preempt_enable_notrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) #endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) * We are reporting a quiescent state on behalf of some other CPU, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) * it is our responsibility to check for and handle potential overflow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) * of the rcu_node ->gp_seq counter with respect to the rcu_data counters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) * After all, the CPU might be in deep idle state, and thus executing no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) * code whatsoever.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) raw_lockdep_assert_held_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) rnp->gp_seq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) WRITE_ONCE(rdp->gpwrap, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) * Snapshot the specified CPU's dynticks counter so that we can later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) * credit them with an implicit quiescent state. Return 1 if this CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) * is in dynticks idle mode, which is an extended quiescent state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) static int dyntick_save_progress_counter(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) rdp->dynticks_snap = rcu_dynticks_snap(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) rcu_gpnum_ovf(rdp->mynode, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * Return true if the specified CPU has passed through a quiescent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * state by virtue of being in or having passed through an dynticks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) * idle state since the last call to dyntick_save_progress_counter()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * for this same CPU, or by virtue of having been offline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) unsigned long jtsq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) bool *rnhqp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) bool *ruqp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) struct rcu_node *rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) * If the CPU passed through or entered a dynticks idle phase with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) * no active irq/NMI handlers, then we can safely pretend that the CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) * already acknowledged the request to pass through a quiescent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * state. Either way, that CPU cannot possibly be in an RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * read-side critical section that started before the beginning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) * of the current RCU grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) if (rcu_dynticks_in_eqs_since(rdp, rdp->dynticks_snap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) rcu_gpnum_ovf(rnp, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) * Complain if a CPU that is considered to be offline from RCU's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) * perspective has not yet reported a quiescent state. After all,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) * the offline CPU should have reported a quiescent state during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) * the CPU-offline process, or, failing that, by rcu_gp_init()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) * if it ran concurrently with either the CPU going offline or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) * last task on a leaf rcu_node structure exiting its RCU read-side
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * critical section while all CPUs corresponding to that structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * are offline. This added warning detects bugs in any of these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * code paths.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) * The rcu_node structure's ->lock is held here, which excludes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) * the relevant portions the CPU-hotplug code, the grace-period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) * initialization code, and the rcu_read_unlock() code paths.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) * For more detail, please refer to the "Hotplug CPU" section
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) * of RCU's Requirements documentation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) if (WARN_ON_ONCE(!(rdp->grpmask & rcu_rnp_online_cpus(rnp)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) bool onl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) struct rcu_node *rnp1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) __func__, rnp->grplo, rnp->grphi, rnp->level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) (long)rnp->gp_seq, (long)rnp->completedqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx ->rcu_gp_init_mask %#lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext, rnp1->rcu_gp_init_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) pr_info("%s %d: %c online: %ld(%d) offline: %ld(%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) __func__, rdp->cpu, ".o"[onl],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) return 1; /* Break things loose after complaining. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) * A CPU running for an extended time within the kernel can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) * delay RCU grace periods: (1) At age jiffies_to_sched_qs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) * set .rcu_urgent_qs, (2) At age 2*jiffies_to_sched_qs, set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) * both .rcu_need_heavy_qs and .rcu_urgent_qs. Note that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) * unsynchronized assignments to the per-CPU rcu_need_heavy_qs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) * variable are safe because the assignments are repeated if this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) * CPU failed to pass through a quiescent state. This code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) * also checks .jiffies_resched in case jiffies_to_sched_qs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) * is set way high.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) jtsq = READ_ONCE(jiffies_to_sched_qs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) if (!READ_ONCE(*rnhqp) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) (time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) time_after(jiffies, rcu_state.jiffies_resched) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) rcu_state.cbovld)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) WRITE_ONCE(*rnhqp, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) /* Store rcu_need_heavy_qs before rcu_urgent_qs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) smp_store_release(ruqp, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) WRITE_ONCE(*ruqp, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) * NO_HZ_FULL CPUs can run in-kernel without rcu_sched_clock_irq!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) * The above code handles this, but only for straight cond_resched().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) * And some in-kernel loops check need_resched() before calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) * cond_resched(), which defeats the above code for CPUs that are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) * running in-kernel with scheduling-clock interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * So hit them over the head with the resched_cpu() hammer!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) if (tick_nohz_full_cpu(rdp->cpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) rcu_state.cbovld)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) WRITE_ONCE(*ruqp, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) resched_cpu(rdp->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) WRITE_ONCE(rdp->last_fqs_resched, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) * If more than halfway to RCU CPU stall-warning time, invoke
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) * resched_cpu() more frequently to try to loosen things up a bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) * Also check to see if the CPU is getting hammered with interrupts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) * but only once per grace period, just to keep the IPIs down to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) * a dull roar.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) if (time_after(jiffies, rcu_state.jiffies_resched)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) if (time_after(jiffies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) READ_ONCE(rdp->last_fqs_resched) + jtsq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) resched_cpu(rdp->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) WRITE_ONCE(rdp->last_fqs_resched, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) if (IS_ENABLED(CONFIG_IRQ_WORK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) (rnp->ffmask & rdp->grpmask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) init_irq_work(&rdp->rcu_iw, rcu_iw_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) atomic_set(&rdp->rcu_iw.flags, IRQ_WORK_HARD_IRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) rdp->rcu_iw_pending = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) rdp->rcu_iw_gp_seq = rnp->gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) /* Trace-event wrapper function for trace_rcu_future_grace_period. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) unsigned long gp_seq_req, const char *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) gp_seq_req, rnp->level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) rnp->grplo, rnp->grphi, s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) * rcu_start_this_gp - Request the start of a particular grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) * @rnp_start: The leaf node of the CPU from which to start.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) * @rdp: The rcu_data corresponding to the CPU from which to start.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) * @gp_seq_req: The gp_seq of the grace period to start.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) * Start the specified grace period, as needed to handle newly arrived
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) * callbacks. The required future grace periods are recorded in each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) * rcu_node structure's ->gp_seq_needed field. Returns true if there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) * is reason to awaken the grace-period kthread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) * The caller must hold the specified rcu_node structure's ->lock, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) * is why the caller is responsible for waking the grace-period kthread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) * Returns true if the GP thread needs to be awakened else false.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) unsigned long gp_seq_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) * Use funnel locking to either acquire the root rcu_node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * structure's lock or bail out if the need for this grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) * has already been recorded -- or if that grace period has in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) * fact already started. If there is already a grace period in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) * progress in a non-leaf node, no recording is needed because the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) * end of the grace period will scan the leaf rcu_node structures.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) * Note that rnp_start->lock must not be released.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) raw_lockdep_assert_held_rcu_node(rnp_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, TPS("Startleaf"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) for (rnp = rnp_start; 1; rnp = rnp->parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) if (rnp != rnp_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) raw_spin_lock_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) if (ULONG_CMP_GE(rnp->gp_seq_needed, gp_seq_req) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) rcu_seq_started(&rnp->gp_seq, gp_seq_req) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) (rnp != rnp_start &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) trace_rcu_this_gp(rnp, rdp, gp_seq_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) TPS("Prestarted"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) goto unlock_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) * We just marked the leaf or internal node, and a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) * grace period is in progress, which means that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) * rcu_gp_cleanup() will see the marking. Bail to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) * reduce contention.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) trace_rcu_this_gp(rnp_start, rdp, gp_seq_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) TPS("Startedleaf"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) goto unlock_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) if (rnp != rnp_start && rnp->parent != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) raw_spin_unlock_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) if (!rnp->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) break; /* At root, and perhaps also leaf. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) /* If GP already in progress, just leave, otherwise start one. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) if (rcu_gp_in_progress()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) goto unlock_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) if (!READ_ONCE(rcu_state.gp_kthread)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) goto unlock_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) ret = true; /* Caller must wake GP kthread. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) unlock_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) /* Push furthest requested GP to leaf node and rcu_data structure. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) WRITE_ONCE(rnp_start->gp_seq_needed, rnp->gp_seq_needed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) if (rnp != rnp_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) raw_spin_unlock_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) * Clean up any old requests for the just-ended grace period. Also return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) * whether any additional grace periods have been requested.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) static bool rcu_future_gp_cleanup(struct rcu_node *rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) bool needmore;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) if (!needmore)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) trace_rcu_this_gp(rnp, rdp, rnp->gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) needmore ? TPS("CleanupMore") : TPS("Cleanup"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) return needmore;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) * Awaken the grace-period kthread. Don't do a self-awaken (unless in an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) * interrupt or softirq handler, in which case we just might immediately
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) * sleep upon return, resulting in a grace-period hang), and don't bother
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) * awakening when there is nothing for the grace-period kthread to do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) * (as in several CPUs raced to awaken, we lost), and finally don't try
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) * to awaken a kthread that has not yet been created. If all those checks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) * are passed, track some debug information and awaken.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) * So why do the self-wakeup when in an interrupt or softirq handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) * in the grace-period kthread's context? Because the kthread might have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) * been interrupted just as it was going to sleep, and just after the final
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) * pre-sleep check of the awaken condition. In this case, a wakeup really
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) * is required, and is therefore supplied.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) static void rcu_gp_kthread_wake(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) if ((current == t && !in_irq() && !in_serving_softirq()) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) !READ_ONCE(rcu_state.gp_flags) || !t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) swake_up_one(&rcu_state.gp_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) * If there is room, assign a ->gp_seq number to any callbacks on this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) * CPU that have not already been assigned. Also accelerate any callbacks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) * that were previously assigned a ->gp_seq number that has since proven
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) * to be too conservative, which can happen if callbacks get assigned a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) * ->gp_seq number while RCU is idle, but with reference to a non-root
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) * rcu_node structure. This function is idempotent, so it does not hurt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) * to call it repeatedly. Returns an flag saying that we should awaken
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) * the RCU grace-period kthread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) * The caller must hold rnp->lock with interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) unsigned long gp_seq_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) rcu_lockdep_assert_cblist_protected(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) raw_lockdep_assert_held_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) if (!rcu_segcblist_pend_cbs(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) * Callbacks are often registered with incomplete grace-period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) * information. Something about the fact that getting exact
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) * information requires acquiring a global lock... RCU therefore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) * makes a conservative estimate of the grace period number at which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) * a given callback will become ready to invoke. The following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) * code checks this estimate and improves it when possible, thus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) * accelerating callback invocation to an earlier grace-period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) * number.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) ret = rcu_start_this_gp(rnp, rdp, gp_seq_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) /* Trace depending on how much we were able to accelerate. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) * Similar to rcu_accelerate_cbs(), but does not require that the leaf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) * rcu_node structure's ->lock be held. It consults the cached value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) * of ->gp_seq_needed in the rcu_data structure, and if that indicates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) * that a new grace-period request be made, invokes rcu_accelerate_cbs()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) * while holding the leaf rcu_node structure's ->lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) unsigned long c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) bool needwake;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) rcu_lockdep_assert_cblist_protected(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) c = rcu_seq_snap(&rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) /* Old request still live, so mark recent callbacks. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) (void)rcu_segcblist_accelerate(&rdp->cblist, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) needwake = rcu_accelerate_cbs(rnp, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) if (needwake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) rcu_gp_kthread_wake();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) * Move any callbacks whose grace period has completed to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) * RCU_DONE_TAIL sublist, then compact the remaining sublists and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) * assign ->gp_seq numbers to any callbacks in the RCU_NEXT_TAIL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) * sublist. This function is idempotent, so it does not hurt to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) * invoke it repeatedly. As long as it is not invoked -too- often...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) * Returns true if the RCU grace-period kthread needs to be awakened.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) * The caller must hold rnp->lock with interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) rcu_lockdep_assert_cblist_protected(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) raw_lockdep_assert_held_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) if (!rcu_segcblist_pend_cbs(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) * Find all callbacks whose ->gp_seq numbers indicate that they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) * are ready to invoke, and put them into the RCU_DONE_TAIL sublist.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) /* Classify any remaining callbacks. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) return rcu_accelerate_cbs(rnp, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) * Move and classify callbacks, but only if doing so won't require
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) * that the RCU grace-period kthread be awakened.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) rcu_lockdep_assert_cblist_protected(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) || !raw_spin_trylock_rcu_node(rnp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) // The grace period cannot end while we hold the rcu_node lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) raw_spin_unlock_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) * In CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels, attempt to generate a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) * quiescent state. This is intended to be invoked when the CPU notices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) * a new grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) static void rcu_strict_gp_check_qs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) * Update CPU-local rcu_data state to record the beginnings and ends of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) * grace periods. The caller must hold the ->lock of the leaf rcu_node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) * structure corresponding to the current CPU, and must have irqs disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) * Returns true if the grace-period kthread needs to be awakened.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) bool need_qs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) rcu_segcblist_is_offloaded(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) raw_lockdep_assert_held_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) if (rdp->gp_seq == rnp->gp_seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) return false; /* Nothing to do. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) /* Handle the ends of any preceding grace periods first. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) unlikely(READ_ONCE(rdp->gpwrap))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) if (!offloaded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) rdp->core_needs_qs = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) if (!offloaded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) if (rdp->core_needs_qs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) /* Now handle the beginnings of any new-to-this-CPU grace periods. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) unlikely(READ_ONCE(rdp->gpwrap))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) * If the current grace period is waiting for this CPU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) * set up to detect a quiescent state, otherwise don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) * go looking for one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) need_qs = !!(rnp->qsmask & rdp->grpmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) rdp->cpu_no_qs.b.norm = need_qs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) rdp->core_needs_qs = need_qs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) zero_cpu_stall_ticks(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) WRITE_ONCE(rdp->gpwrap, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) rcu_gpnum_ovf(rnp, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) static void note_gp_changes(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) bool needwake;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) !raw_spin_trylock_rcu_node(rnp)) { /* irqs already off, so later. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) needwake = __note_gp_changes(rnp, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) rcu_strict_gp_check_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) if (needwake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) rcu_gp_kthread_wake();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) static void rcu_gp_slow(int delay)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) if (delay > 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) !(rcu_seq_ctr(rcu_state.gp_seq) %
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) schedule_timeout_idle(delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) static unsigned long sleep_duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) /* Allow rcutorture to stall the grace-period kthread. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) void rcu_gp_set_torture_wait(int duration)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST) && duration > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) WRITE_ONCE(sleep_duration, duration);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) EXPORT_SYMBOL_GPL(rcu_gp_set_torture_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) /* Actually implement the aforementioned wait. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) static void rcu_gp_torture_wait(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) unsigned long duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) duration = xchg(&sleep_duration, 0UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) if (duration > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) pr_alert("%s: Waiting %lu jiffies\n", __func__, duration);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) schedule_timeout_idle(duration);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) pr_alert("%s: Wait complete\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) * Handler for on_each_cpu() to invoke the target CPU's RCU core
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) * processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) static void rcu_strict_gp_boundary(void *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) invoke_rcu_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) * Initialize a new grace period. Return false if no grace period required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) static bool rcu_gp_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) unsigned long oldmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) struct rcu_node *rnp = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) WRITE_ONCE(rcu_state.gp_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) raw_spin_lock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) if (!READ_ONCE(rcu_state.gp_flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) /* Spurious wakeup, tell caller to go back to sleep. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) if (WARN_ON_ONCE(rcu_gp_in_progress())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) * Grace period already in progress, don't start another.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) * Not supposed to be able to happen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) /* Advance to a new grace period and initialize state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) record_gp_stall_check_time();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) /* Record GP times before starting GP, hence rcu_seq_start(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) rcu_seq_start(&rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) * Apply per-leaf buffered online and offline operations to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) * the rcu_node tree. Note that this new grace period need not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) * wait for subsequent online CPUs, and that RCU hooks in the CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) * offlining path, when combined with checks in this function,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) * will handle CPUs that are currently going offline or that will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) * go offline later. Please also refer to "Hotplug CPU" section
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) * of RCU's Requirements documentation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) rcu_state.gp_state = RCU_GP_ONOFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) rcu_for_each_leaf_node(rnp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) raw_spin_lock(&rcu_state.ofl_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) raw_spin_lock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) !rnp->wait_blkd_tasks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) /* Nothing to do on this leaf rcu_node structure. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) raw_spin_unlock(&rcu_state.ofl_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) /* Record old state, apply changes to ->qsmaskinit field. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) oldmask = rnp->qsmaskinit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) rnp->qsmaskinit = rnp->qsmaskinitnext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) /* If zero-ness of ->qsmaskinit changed, propagate up tree. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) if (!oldmask != !rnp->qsmaskinit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) if (!oldmask) { /* First online CPU for rcu_node. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) if (!rnp->wait_blkd_tasks) /* Ever offline? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) rcu_init_new_rnp(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) } else if (rcu_preempt_has_tasks(rnp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) rnp->wait_blkd_tasks = true; /* blocked tasks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) } else { /* Last offline CPU and can propagate. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) rcu_cleanup_dead_rnp(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) * If all waited-on tasks from prior grace period are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) * done, and if all this rcu_node structure's CPUs are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) * still offline, propagate up the rcu_node tree and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) * clear ->wait_blkd_tasks. Otherwise, if one of this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) * rcu_node structure's CPUs has since come back online,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) * simply clear ->wait_blkd_tasks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) if (rnp->wait_blkd_tasks &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) (!rcu_preempt_has_tasks(rnp) || rnp->qsmaskinit)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) rnp->wait_blkd_tasks = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) if (!rnp->qsmaskinit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) rcu_cleanup_dead_rnp(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) raw_spin_unlock(&rcu_state.ofl_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) rcu_gp_slow(gp_preinit_delay); /* Races with CPU hotplug. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) * Set the quiescent-state-needed bits in all the rcu_node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) * structures for all currently online CPUs in breadth-first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) * order, starting from the root rcu_node structure, relying on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) * layout of the tree within the rcu_state.node[] array. Note that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) * other CPUs will access only the leaves of the hierarchy, thus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) * seeing that no grace period is in progress, at least until the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) * corresponding leaf node has been initialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) * The grace period cannot complete until the initialization
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) * process finishes, because this kthread handles both.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) rcu_state.gp_state = RCU_GP_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) rcu_for_each_node_breadth_first(rnp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) rcu_gp_slow(gp_init_delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) rcu_preempt_check_blocked_tasks(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) rnp->qsmask = rnp->qsmaskinit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) if (rnp == rdp->mynode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) (void)__note_gp_changes(rnp, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) rcu_preempt_boost_start_gp(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) rnp->level, rnp->grplo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) rnp->grphi, rnp->qsmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) /* Quiescent states for tasks on any now-offline CPUs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) mask = rnp->qsmask & ~rnp->qsmaskinitnext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) rnp->rcu_gp_init_mask = mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) if ((mask || rnp->wait_blkd_tasks) && rcu_is_leaf_node(rnp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) cond_resched_tasks_rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) WRITE_ONCE(rcu_state.gp_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) // If strict, make all CPUs aware of new grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) * Helper function for swait_event_idle_exclusive() wakeup at force-quiescent-state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) * time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) static bool rcu_gp_fqs_check_wake(int *gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) struct rcu_node *rnp = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) // If under overload conditions, force an immediate FQS scan.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) if (*gfp & RCU_GP_FLAG_OVLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) // Someone like call_rcu() requested a force-quiescent-state scan.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) *gfp = READ_ONCE(rcu_state.gp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) if (*gfp & RCU_GP_FLAG_FQS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) // The current grace period has completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) * Do one round of quiescent-state forcing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) static void rcu_gp_fqs(bool first_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) struct rcu_node *rnp = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) WRITE_ONCE(rcu_state.gp_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) if (first_time) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) /* Collect dyntick-idle snapshots. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) force_qs_rnp(dyntick_save_progress_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) /* Handle dyntick-idle and offline CPUs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) force_qs_rnp(rcu_implicit_dynticks_qs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) /* Clear flag to prevent immediate re-entry. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) raw_spin_lock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) WRITE_ONCE(rcu_state.gp_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) * Loop doing repeated quiescent-state forcing until the grace period ends.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) static void rcu_gp_fqs_loop(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) bool first_gp_fqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) int gf = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) struct rcu_node *rnp = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) first_gp_fqs = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) j = READ_ONCE(jiffies_till_first_fqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) if (rcu_state.cbovld)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) gf = RCU_GP_FLAG_OVLD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) rcu_state.jiffies_force_qs = jiffies + j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) jiffies + (j ? 3 * j : 2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) TPS("fqswait"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) rcu_state.gp_state = RCU_GP_WAIT_FQS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) ret = swait_event_idle_timeout_exclusive(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) rcu_gp_torture_wait();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) rcu_state.gp_state = RCU_GP_DOING_FQS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) /* Locking provides needed memory barriers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) /* If grace period done, leave loop. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) if (!READ_ONCE(rnp->qsmask) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) !rcu_preempt_blocked_readers_cgp(rnp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) /* If time for quiescent-state forcing, do it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) if (!time_after(rcu_state.jiffies_force_qs, jiffies) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) (gf & (RCU_GP_FLAG_FQS | RCU_GP_FLAG_OVLD))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) TPS("fqsstart"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) rcu_gp_fqs(first_gp_fqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) gf = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) if (first_gp_fqs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) first_gp_fqs = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) TPS("fqsend"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) cond_resched_tasks_rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) WRITE_ONCE(rcu_state.gp_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) ret = 0; /* Force full wait till next FQS. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) j = READ_ONCE(jiffies_till_next_fqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) /* Deal with stray signal. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) cond_resched_tasks_rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) WRITE_ONCE(rcu_state.gp_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) WARN_ON(signal_pending(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) TPS("fqswaitsig"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) ret = 1; /* Keep old FQS timing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) j = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) if (time_after(jiffies, rcu_state.jiffies_force_qs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) j = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) j = rcu_state.jiffies_force_qs - j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) gf = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) * Clean up after the old grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) static void rcu_gp_cleanup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) bool needgp = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) unsigned long gp_duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) unsigned long new_gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) bool offloaded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) struct rcu_node *rnp = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) struct swait_queue_head *sq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) WRITE_ONCE(rcu_state.gp_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) raw_spin_lock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) rcu_state.gp_end = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) gp_duration = rcu_state.gp_end - rcu_state.gp_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) if (gp_duration > rcu_state.gp_max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) rcu_state.gp_max = gp_duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) * We know the grace period is complete, but to everyone else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) * it appears to still be ongoing. But it is also the case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) * that to everyone else it looks like there is nothing that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) * they can do to advance the grace period. It is therefore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) * safe for us to drop the lock in order to mark the grace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) * period as completed in all of the rcu_node structures.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) * Propagate new ->gp_seq value to rcu_node structures so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) * other CPUs don't have to wait until the start of the next grace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) * period to process their callbacks. This also avoids some nasty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) * RCU grace-period initialization races by forcing the end of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) * the current grace period to be completely recorded in all of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) * the rcu_node structures before the beginning of the next grace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) * period is recorded in any of the rcu_node structures.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) new_gp_seq = rcu_state.gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) rcu_seq_end(&new_gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) rcu_for_each_node_breadth_first(rnp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) raw_spin_lock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) dump_blkd_tasks(rnp, 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) WARN_ON_ONCE(rnp->qsmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) WRITE_ONCE(rnp->gp_seq, new_gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) if (rnp == rdp->mynode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) needgp = __note_gp_changes(rnp, rdp) || needgp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) /* smp_mb() provided by prior unlock-lock pair. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) needgp = rcu_future_gp_cleanup(rnp) || needgp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) // Reset overload indication for CPUs no longer overloaded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) if (rcu_is_leaf_node(rnp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) for_each_leaf_node_cpu_mask(rnp, cpu, rnp->cbovldmask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) check_cb_ovld_locked(rdp, rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) sq = rcu_nocb_gp_get(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) rcu_nocb_gp_cleanup(sq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) cond_resched_tasks_rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) WRITE_ONCE(rcu_state.gp_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) rcu_gp_slow(gp_cleanup_delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) rnp = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) raw_spin_lock_irq_rcu_node(rnp); /* GP before ->gp_seq update. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) /* Declare grace period done, trace first to use old GP number. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) rcu_seq_end(&rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) rcu_state.gp_state = RCU_GP_IDLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) /* Check for GP requests since above loop. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) TPS("CleanupMore"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) needgp = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) /* Advance CBs to reduce false positives below. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) rcu_segcblist_is_offloaded(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) trace_rcu_grace_period(rcu_state.name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) rcu_state.gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) TPS("newreq"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) WRITE_ONCE(rcu_state.gp_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) rcu_state.gp_flags & RCU_GP_FLAG_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) raw_spin_unlock_irq_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) // If strict, make all CPUs aware of the end of the old grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) on_each_cpu(rcu_strict_gp_boundary, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) * Body of kthread that handles grace periods.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) static int __noreturn rcu_gp_kthread(void *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) rcu_bind_gp_kthread();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) /* Handle grace-period start. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) TPS("reqwait"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) rcu_state.gp_state = RCU_GP_WAIT_GPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) swait_event_idle_exclusive(rcu_state.gp_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) READ_ONCE(rcu_state.gp_flags) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) RCU_GP_FLAG_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) rcu_gp_torture_wait();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) rcu_state.gp_state = RCU_GP_DONE_GPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) /* Locking provides needed memory barrier. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) if (rcu_gp_init())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) cond_resched_tasks_rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) WRITE_ONCE(rcu_state.gp_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) WARN_ON(signal_pending(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) TPS("reqwaitsig"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) /* Handle quiescent-state forcing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) rcu_gp_fqs_loop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) /* Handle grace-period end. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) rcu_state.gp_state = RCU_GP_CLEANUP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) rcu_gp_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) rcu_state.gp_state = RCU_GP_CLEANED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) * Report a full set of quiescent states to the rcu_state data structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) * Invoke rcu_gp_kthread_wake() to awaken the grace-period kthread if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) * another grace period is required. Whether we wake the grace-period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) * kthread or it awakens itself for the next round of quiescent-state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) * forcing, that kthread will clean up after the just-completed grace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) * period. Note that the caller must hold rnp->lock, which is released
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) * before return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) static void rcu_report_qs_rsp(unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) __releases(rcu_get_root()->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) raw_lockdep_assert_held_rcu_node(rcu_get_root());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) WARN_ON_ONCE(!rcu_gp_in_progress());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) WRITE_ONCE(rcu_state.gp_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) rcu_gp_kthread_wake();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) * Similar to rcu_report_qs_rdp(), for which it is a helper function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) * Allows quiescent states for a group of CPUs to be reported at one go
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) * to the specified rcu_node structure, though all the CPUs in the group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) * must be represented by the same rcu_node structure (which need not be a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) * leaf rcu_node structure, though it often will be). The gps parameter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) * is the grace-period snapshot, which means that the quiescent states
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) * are valid only if rnp->gp_seq is equal to gps. That structure's lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) * must be held upon entry, and it is released before return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) * As a special case, if mask is zero, the bit-already-cleared check is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) * disabled. This allows propagating quiescent state due to resumed tasks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) * during grace-period initialization.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) unsigned long gps, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) __releases(rnp->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) unsigned long oldmask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) struct rcu_node *rnp_c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) raw_lockdep_assert_held_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) /* Walk up the rcu_node hierarchy. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) * Our bit has already been cleared, or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) * relevant grace period is already over, so done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) rcu_preempt_blocked_readers_cgp(rnp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) WRITE_ONCE(rnp->qsmask, rnp->qsmask & ~mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) mask, rnp->qsmask, rnp->level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) rnp->grplo, rnp->grphi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) !!rnp->gp_tasks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) /* Other bits still set at this level, so done. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) rnp->completedqs = rnp->gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) mask = rnp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) if (rnp->parent == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) /* No more levels. Exit loop holding root lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) rnp_c = rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) rnp = rnp->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) oldmask = READ_ONCE(rnp_c->qsmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) * Get here if we are the last CPU to pass through a quiescent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) * state for this grace period. Invoke rcu_report_qs_rsp()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) * to clean up and start the next grace period if one is needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) rcu_report_qs_rsp(flags); /* releases rnp->lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) * Record a quiescent state for all tasks that were previously queued
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) * on the specified rcu_node structure and that were blocking the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) * RCU grace period. The caller must hold the corresponding rnp->lock with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) * irqs disabled, and this lock is released upon return, but irqs remain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) * disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) static void __maybe_unused
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) __releases(rnp->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) unsigned long gps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) struct rcu_node *rnp_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) raw_lockdep_assert_held_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_RCU)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) rnp->qsmask != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) return; /* Still need more quiescent states! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) rnp->completedqs = rnp->gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) rnp_p = rnp->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) if (rnp_p == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) * Only one rcu_node structure in the tree, so don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) * try to report up to its nonexistent parent!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) rcu_report_qs_rsp(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) /* Report up the rest of the hierarchy, tracking current ->gp_seq. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) gps = rnp->gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) mask = rnp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) raw_spin_lock_rcu_node(rnp_p); /* irqs already disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) rcu_report_qs_rnp(mask, rnp_p, gps, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) * Record a quiescent state for the specified CPU to that CPU's rcu_data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) * structure. This must be called from the specified CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) rcu_report_qs_rdp(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) bool needwake = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) rcu_segcblist_is_offloaded(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) WARN_ON_ONCE(rdp->cpu != smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) rdp->gpwrap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) * The grace period in which this quiescent state was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) * recorded has ended, so don't report it upwards.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) * We will instead need a new quiescent state that lies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) * within the current grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) mask = rdp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) rdp->core_needs_qs = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) if ((rnp->qsmask & mask) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) * This GP can't end until cpu checks in, so all of our
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) * callbacks can be processed during the next GP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) if (!offloaded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) needwake = rcu_accelerate_cbs(rnp, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) rcu_disable_urgency_upon_qs(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) /* ^^^ Released rnp->lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) if (needwake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) rcu_gp_kthread_wake();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) * Check to see if there is a new grace period of which this CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) * is not yet aware, and if so, set up local rcu_data state for it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) * Otherwise, see if this CPU has just passed through its first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) * quiescent state for this grace period, and record that fact if so.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) rcu_check_quiescent_state(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) /* Check for grace-period ends and beginnings. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) note_gp_changes(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) * Does this CPU still need to do its part for current grace period?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) * If no, return and let the other CPUs do their part as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) if (!rdp->core_needs_qs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) * Was there a quiescent state since the beginning of the grace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) * period? If no, then exit and wait for the next call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) if (rdp->cpu_no_qs.b.norm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) * Tell RCU we are done (but rcu_report_qs_rdp() will be the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) * judge of that).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) rcu_report_qs_rdp(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) * Near the end of the offline process. Trace the fact that this CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) * is going offline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) int rcutree_dying_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) bool blkd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) struct rcu_node *rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) blkd = !!(rnp->qsmask & rdp->grpmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) * All CPUs for the specified rcu_node structure have gone offline,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) * and all tasks that were preempted within an RCU read-side critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) * section while running on one of those CPUs have since exited their RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) * read-side critical section. Some other CPU is reporting this fact with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) * the specified rcu_node structure's ->lock held and interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) * This function therefore goes up the tree of rcu_node structures,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) * clearing the corresponding bits in the ->qsmaskinit fields. Note that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) * the leaf rcu_node structure's ->qsmaskinit field has already been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) * updated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) * This function does check that the specified rcu_node structure has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) * all CPUs offline and no blocked tasks, so it is OK to invoke it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) * prematurely. That said, invoking it after the fact will cost you
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) * a needless lock acquisition. So once it has done its work, don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) * invoke it again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) struct rcu_node *rnp = rnp_leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) raw_lockdep_assert_held_rcu_node(rnp_leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) WARN_ON_ONCE(rnp_leaf->qsmaskinit) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) WARN_ON_ONCE(rcu_preempt_has_tasks(rnp_leaf)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) mask = rnp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) rnp = rnp->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) if (!rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) rnp->qsmaskinit &= ~mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) /* Between grace periods, so better already be zero! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) WARN_ON_ONCE(rnp->qsmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) if (rnp->qsmaskinit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) raw_spin_unlock_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) /* irqs remain disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) * The CPU has been completely removed, and some other CPU is reporting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) * this fact from process context. Do the remainder of the cleanup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) * There can only be one CPU hotplug operation at a time, so no need for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) * explicit locking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) int rcutree_dead_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) /* Adjust any no-longer-needed kthreads. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) rcu_boost_kthread_setaffinity(rnp, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) /* Do any needed no-CB deferred wakeups from this CPU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) do_nocb_deferred_wakeup(per_cpu_ptr(&rcu_data, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) // Stop-machine done, so allow nohz_full to disable tick.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) tick_dep_clear(TICK_DEP_BIT_RCU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) * Invoke any RCU callbacks that have made it to the end of their grace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) * period. Thottle as specified by rdp->blimit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) static void rcu_do_batch(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) int div;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) rcu_segcblist_is_offloaded(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) struct rcu_head *rhp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) long bl, count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) long pending, tlimit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) /* If no callbacks are ready, just return. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) trace_rcu_batch_start(rcu_state.name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) rcu_segcblist_n_cbs(&rdp->cblist), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) trace_rcu_batch_end(rcu_state.name, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) !rcu_segcblist_empty(&rdp->cblist),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) need_resched(), is_idle_task(current),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) rcu_is_callbacks_kthread());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) * Extract the list of ready callbacks, disabling to prevent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) * races with call_rcu() from interrupt handlers. Leave the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) * callback counts, as rcu_barrier() needs to be conservative.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) rcu_nocb_lock(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) pending = rcu_segcblist_n_cbs(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) div = READ_ONCE(rcu_divisor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) bl = max(rdp->blimit, pending >> div);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) if (unlikely(bl > 100)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) long rrn = READ_ONCE(rcu_resched_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) tlimit = local_clock() + rrn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) trace_rcu_batch_start(rcu_state.name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) rcu_segcblist_n_cbs(&rdp->cblist), bl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) if (offloaded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) rcu_nocb_unlock_irqrestore(rdp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) /* Invoke callbacks. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) tick_dep_set_task(current, TICK_DEP_BIT_RCU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) rhp = rcu_cblist_dequeue(&rcl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) rcu_callback_t f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) debug_rcu_head_unqueue(rhp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) rcu_lock_acquire(&rcu_callback_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) trace_rcu_invoke_callback(rcu_state.name, rhp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) f = rhp->func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) f(rhp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) rcu_lock_release(&rcu_callback_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) * Stop only if limit reached and CPU has something to do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) * Note: The rcl structure counts down from zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) if (-rcl.len >= bl && !offloaded &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) (need_resched() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) if (unlikely(tlimit)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) /* only call local_clock() every 32 callbacks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) if (likely((-rcl.len & 31) || local_clock() < tlimit))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) /* Exceeded the time limit, so leave. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) if (offloaded) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) WARN_ON_ONCE(in_serving_softirq());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) lockdep_assert_irqs_enabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) cond_resched_tasks_rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) lockdep_assert_irqs_enabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) rcu_nocb_lock(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) count = -rcl.len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) rdp->n_cbs_invoked += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) is_idle_task(current), rcu_is_callbacks_kthread());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) /* Update counts and requeue any remaining callbacks. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) smp_mb(); /* List handling before counting for rcu_barrier(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) rcu_segcblist_insert_count(&rdp->cblist, &rcl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) /* Reinstate batch limit if we have worked down the excess. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) count = rcu_segcblist_n_cbs(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) rdp->blimit = blimit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) if (count == 0 && rdp->qlen_last_fqs_check != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) rdp->qlen_last_fqs_check = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) } else if (count < rdp->qlen_last_fqs_check - qhimark)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) rdp->qlen_last_fqs_check = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) * The following usually indicates a double call_rcu(). To track
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) * this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) count != 0 && rcu_segcblist_empty(&rdp->cblist));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) rcu_nocb_unlock_irqrestore(rdp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) /* Re-invoke RCU core processing if there are callbacks remaining. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) invoke_rcu_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) * This function is invoked from each scheduling-clock interrupt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) * and checks to see if this CPU is in a non-context-switch quiescent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) * state, for example, user mode or idle loop. It also schedules RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) * core processing. If the current grace period has gone on too long,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) * it will ask the scheduler to manufacture a context switch for the sole
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) * purpose of providing a providing the needed quiescent state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) void rcu_sched_clock_irq(int user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) trace_rcu_utilization(TPS("Start scheduler-tick"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) raw_cpu_inc(rcu_data.ticks_this_gp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) /* The load-acquire pairs with the store-release setting to true. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) /* Idle and userspace execution already are quiescent states. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) if (!rcu_is_cpu_rrupt_from_idle() && !user) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) set_tsk_need_resched(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) set_preempt_need_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) __this_cpu_write(rcu_data.rcu_urgent_qs, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) rcu_flavor_sched_clock_irq(user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) if (rcu_pending(user))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) invoke_rcu_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) trace_rcu_utilization(TPS("End scheduler-tick"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) * Scan the leaf rcu_node structures. For each structure on which all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) * CPUs have reported a quiescent state and on which there are tasks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) * blocking the current grace period, initiate RCU priority boosting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) * Otherwise, invoke the specified function to check dyntick state for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) * each CPU that has not yet reported a quiescent state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) static void force_qs_rnp(int (*f)(struct rcu_data *rdp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) rcu_state.cbovld = rcu_state.cbovldnext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) rcu_state.cbovldnext = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) rcu_for_each_leaf_node(rnp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) cond_resched_tasks_rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) rcu_state.cbovldnext |= !!rnp->cbovldmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) if (rnp->qsmask == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) if (rcu_preempt_blocked_readers_cgp(rnp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) * No point in scanning bits because they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) * are all zero. But we might need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) * priority-boost blocked readers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) rcu_initiate_boost(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) /* rcu_initiate_boost() releases rnp->lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) for_each_leaf_node_cpu_mask(rnp, cpu, rnp->qsmask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) if (f(rdp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) mask |= rdp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) rcu_disable_urgency_upon_qs(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) if (mask != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) /* Idle/offline CPUs, report (releases rnp->lock). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) /* Nothing to do here, so just drop the lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) * Force quiescent states on reluctant CPUs, and also detect which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) * CPUs are in dyntick-idle mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) void rcu_force_quiescent_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) struct rcu_node *rnp_old = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) /* Funnel through hierarchy to reduce memory contention. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) rnp = __this_cpu_read(rcu_data.mynode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) for (; rnp != NULL; rnp = rnp->parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) !raw_spin_trylock(&rnp->fqslock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) if (rnp_old != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) raw_spin_unlock(&rnp_old->fqslock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) rnp_old = rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) /* rnp_old == rcu_get_root(), rnp == NULL. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) /* Reached the root of the rcu_node tree, acquire lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) raw_spin_lock_irqsave_rcu_node(rnp_old, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) raw_spin_unlock(&rnp_old->fqslock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) return; /* Someone beat us to it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) WRITE_ONCE(rcu_state.gp_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) rcu_gp_kthread_wake();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) // Workqueue handler for an RCU reader for kernels enforcing struct RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) // grace periods.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) static void strict_work_handler(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) /* Perform RCU core processing work for the current CPU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) static __latent_entropy void rcu_core(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) struct rcu_node *rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) rcu_segcblist_is_offloaded(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) if (cpu_is_offline(smp_processor_id()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) trace_rcu_utilization(TPS("Start RCU core"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) WARN_ON_ONCE(!rdp->beenonline);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) /* Report any deferred quiescent states if preemption enabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) if (!(preempt_count() & PREEMPT_MASK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) rcu_preempt_deferred_qs(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) } else if (rcu_preempt_need_deferred_qs(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) set_tsk_need_resched(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) set_preempt_need_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) /* Update RCU state based on any recent quiescent states. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) rcu_check_quiescent_state(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) /* No grace period and unregistered callbacks? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) if (!rcu_gp_in_progress() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) rcu_accelerate_cbs_unlocked(rnp, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) /* If there are callbacks ready, invoke them. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) likely(READ_ONCE(rcu_scheduler_fully_active)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) rcu_do_batch(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) /* Do any needed deferred wakeups of rcuo kthreads. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) do_nocb_deferred_wakeup(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) trace_rcu_utilization(TPS("End RCU core"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) // If strict GPs, schedule an RCU reader in a clean environment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) static void rcu_core_si(struct softirq_action *h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) rcu_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) static void rcu_wake_cond(struct task_struct *t, int status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) * If the thread is yielding, only wake it when this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) * is invoked from idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) wake_up_process(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) static void invoke_rcu_core_kthread(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) struct task_struct *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) if (t != NULL && t != current)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) * Wake up this CPU's rcuc kthread to do RCU core processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) static void invoke_rcu_core(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) if (!cpu_online(smp_processor_id()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) if (use_softirq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) raise_softirq(RCU_SOFTIRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) invoke_rcu_core_kthread();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) static void rcu_cpu_kthread_park(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) static int rcu_cpu_kthread_should_run(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) return __this_cpu_read(rcu_data.rcu_cpu_has_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) * Per-CPU kernel thread that invokes RCU callbacks. This replaces
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) * the RCU softirq used in configurations of RCU that do not support RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) * priority boosting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) static void rcu_cpu_kthread(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) int spincnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) trace_rcu_utilization(TPS("Start CPU kthread@rcu_run"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) for (spincnt = 0; spincnt < 10; spincnt++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) *statusp = RCU_KTHREAD_RUNNING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) work = *workp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) *workp = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) if (work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) rcu_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) if (*workp == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) *statusp = RCU_KTHREAD_WAITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) *statusp = RCU_KTHREAD_YIELDING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) schedule_timeout_idle(2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) *statusp = RCU_KTHREAD_WAITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) static struct smp_hotplug_thread rcu_cpu_thread_spec = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) .store = &rcu_data.rcu_cpu_kthread_task,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) .thread_should_run = rcu_cpu_kthread_should_run,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) .thread_fn = rcu_cpu_kthread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) .thread_comm = "rcuc/%u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) .setup = rcu_cpu_kthread_setup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) .park = rcu_cpu_kthread_park,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) * Spawn per-CPU RCU core processing kthreads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) static int __init rcu_spawn_core_kthreads(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) if (!IS_ENABLED(CONFIG_RCU_BOOST) && use_softirq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) "%s: Could not start rcuc kthread, OOM is now expected behavior\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) * Handle any core-RCU processing required by a call_rcu() invocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) * If called from an extended quiescent state, invoke the RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) * core in order to force a re-evaluation of RCU's idleness.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) if (!rcu_is_watching())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) invoke_rcu_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) * Force the grace period if too many callbacks or too long waiting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) * Enforce hysteresis, and don't invoke rcu_force_quiescent_state()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) * if some other CPU has recently done so. Also, don't bother
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) * invoking rcu_force_quiescent_state() if the newly enqueued callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) * is the only one waiting for a grace period to complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) rdp->qlen_last_fqs_check + qhimark)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) /* Are we ignoring a completed grace period? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) note_gp_changes(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) /* Start a new grace period if one not already started. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) if (!rcu_gp_in_progress()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) rcu_accelerate_cbs_unlocked(rdp->mynode, rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) /* Give the grace period a kick. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) rdp->blimit = DEFAULT_MAX_RCU_BLIMIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) rcu_segcblist_first_pend_cb(&rdp->cblist) != head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) rcu_force_quiescent_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) * RCU callback function to leak a callback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) static void rcu_leak_callback(struct rcu_head *rhp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) * Check and if necessary update the leaf rcu_node structure's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) * ->cbovldmask bit corresponding to the current CPU based on that CPU's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) * number of queued RCU callbacks. The caller must hold the leaf rcu_node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) * structure's ->lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) raw_lockdep_assert_held_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) if (qovld_calc <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) return; // Early boot and wildcard value set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask | rdp->grpmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) * Check and if necessary update the leaf rcu_node structure's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) * ->cbovldmask bit corresponding to the current CPU based on that CPU's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) * number of queued RCU callbacks. No locks need be held, but the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) * caller must have disabled interrupts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) * Note that this function ignores the possibility that there are a lot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) * of callbacks all of which have already seen the end of their respective
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) * grace periods. This omission is due to the need for no-CBs CPUs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) * be holding ->nocb_lock to do this check, which is too heavy for a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) * common-case operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) static void check_cb_ovld(struct rcu_data *rdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) struct rcu_node *const rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) if (qovld_calc <= 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) return; // Early boot wildcard value or already set correctly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) raw_spin_lock_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) check_cb_ovld_locked(rdp, rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) raw_spin_unlock_rcu_node(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) /* Helper function for call_rcu() and friends. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) __call_rcu(struct rcu_head *head, rcu_callback_t func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) bool was_alldone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) /* Misaligned rcu_head! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) if (debug_rcu_head_queue(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) * Probable double call_rcu(), so leak the callback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) * Use rcu:rcu_callback trace event to find the previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) * time callback was passed to __call_rcu().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pS()!!!\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) head, head->func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) WRITE_ONCE(head->func, rcu_leak_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) head->func = func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) head->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) kasan_record_aux_stack(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) /* Add the callback to our list. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) // This can trigger due to call_rcu() from offline CPU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) WARN_ON_ONCE(rcu_scheduler_active != RCU_SCHEDULER_INACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) WARN_ON_ONCE(!rcu_is_watching());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) // Very early boot, before rcu_init(). Initialize if needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) // and then drop through to queue the callback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) if (rcu_segcblist_empty(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) rcu_segcblist_init(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) check_cb_ovld(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) return; // Enqueued onto ->nocb_bypass, so just leave.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) rcu_segcblist_enqueue(&rdp->cblist, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) if (__is_kvfree_rcu_offset((unsigned long)func))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) trace_rcu_kvfree_callback(rcu_state.name, head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) (unsigned long)func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) rcu_segcblist_n_cbs(&rdp->cblist));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) trace_rcu_callback(rcu_state.name, head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) rcu_segcblist_n_cbs(&rdp->cblist));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) /* Go handle any RCU core processing required. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) __call_rcu_core(rdp, head, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) * call_rcu() - Queue an RCU callback for invocation after a grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) * @head: structure to be used for queueing the RCU updates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) * @func: actual callback function to be invoked after the grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) * The callback function will be invoked some time after a full grace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) * period elapses, in other words after all pre-existing RCU read-side
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) * critical sections have completed. However, the callback function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) * might well execute concurrently with RCU read-side critical sections
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) * that started after call_rcu() was invoked. RCU read-side critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) * sections are delimited by rcu_read_lock() and rcu_read_unlock(), and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) * may be nested. In addition, regions of code across which interrupts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) * preemption, or softirqs have been disabled also serve as RCU read-side
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) * critical sections. This includes hardware interrupt handlers, softirq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) * handlers, and NMI handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) * Note that all CPUs must agree that the grace period extended beyond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) * all pre-existing RCU read-side critical section. On systems with more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) * than one CPU, this means that when "func()" is invoked, each CPU is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) * guaranteed to have executed a full memory barrier since the end of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) * last RCU read-side critical section whose beginning preceded the call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) * to call_rcu(). It also means that each CPU executing an RCU read-side
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) * critical section that continues beyond the start of "func()" must have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) * executed a memory barrier after the call_rcu() but before the beginning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) * of that RCU read-side critical section. Note that these guarantees
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) * include CPUs that are offline, idle, or executing in user mode, as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) * well as CPUs that are executing in the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) * resulting RCU callback function "func()", then both CPU A and CPU B are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) * guaranteed to execute a full memory barrier during the time interval
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) * between the call to call_rcu() and the invocation of "func()" -- even
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) * if CPU A and CPU B are the same CPU (but again only if the system has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) * more than one CPU).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) void call_rcu(struct rcu_head *head, rcu_callback_t func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) __call_rcu(head, func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) EXPORT_SYMBOL_GPL(call_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) /* Maximum number of jiffies to wait before draining a batch. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) #define KFREE_DRAIN_JIFFIES (HZ / 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) #define KFREE_N_BATCHES 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) #define FREE_N_CHANNELS 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) * struct kvfree_rcu_bulk_data - single block to store kvfree_rcu() pointers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) * @nr_records: Number of active pointers in the array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) * @next: Next bulk object in the block chain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) * @records: Array of the kvfree_rcu() pointers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) struct kvfree_rcu_bulk_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) unsigned long nr_records;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) struct kvfree_rcu_bulk_data *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) void *records[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) * This macro defines how many entries the "records" array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) * will contain. It is based on the fact that the size of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) * kvfree_rcu_bulk_data structure becomes exactly one page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) #define KVFREE_BULK_MAX_ENTR \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) ((PAGE_SIZE - sizeof(struct kvfree_rcu_bulk_data)) / sizeof(void *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) * @head_free: List of kfree_rcu() objects waiting for a grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) * @bkvhead_free: Bulk-List of kvfree_rcu() objects waiting for a grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) * @krcp: Pointer to @kfree_rcu_cpu structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) struct kfree_rcu_cpu_work {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) struct rcu_work rcu_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) struct rcu_head *head_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) struct kvfree_rcu_bulk_data *bkvhead_free[FREE_N_CHANNELS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) struct kfree_rcu_cpu *krcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) * @head: List of kfree_rcu() objects not yet waiting for a grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) * @bkvhead: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) * @lock: Synchronize access to this structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) * @monitor_todo: Tracks whether a @monitor_work delayed work is pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) * @initialized: The @rcu_work fields have been initialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) * @count: Number of objects for which GP not started
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) * @bkvcache:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) * A simple cache list that contains objects for reuse purpose.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) * In order to save some per-cpu space the list is singular.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) * Even though it is lockless an access has to be protected by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) * per-cpu lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) * @page_cache_work: A work to refill the cache when it is empty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) * @work_in_progress: Indicates that page_cache_work is running
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) * @hrtimer: A hrtimer for scheduling a page_cache_work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) * @nr_bkv_objs: number of allocated objects at @bkvcache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) * This is a per-CPU structure. The reason that it is not included in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) * the rcu_data structure is to permit this code to be extracted from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) * the RCU files. Such extraction could allow further optimization of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) * the interactions with the slab allocators.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) struct kfree_rcu_cpu {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) struct rcu_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) raw_spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) struct delayed_work monitor_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) bool monitor_todo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) bool initialized;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) struct work_struct page_cache_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) atomic_t work_in_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) struct hrtimer hrtimer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) struct llist_head bkvcache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) int nr_bkv_objs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) .lock = __RAW_SPIN_LOCK_UNLOCKED(krc.lock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) debug_rcu_bhead_unqueue(struct kvfree_rcu_bulk_data *bhead)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) for (i = 0; i < bhead->nr_records; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) debug_rcu_head_unqueue((struct rcu_head *)(bhead->records[i]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) static inline struct kfree_rcu_cpu *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) krc_this_cpu_lock(unsigned long *flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) struct kfree_rcu_cpu *krcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) local_irq_save(*flags); // For safely calling this_cpu_ptr().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) krcp = this_cpu_ptr(&krc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) raw_spin_lock(&krcp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) return krcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) raw_spin_unlock(&krcp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) static inline struct kvfree_rcu_bulk_data *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) get_cached_bnode(struct kfree_rcu_cpu *krcp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) if (!krcp->nr_bkv_objs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) krcp->nr_bkv_objs--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) return (struct kvfree_rcu_bulk_data *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) llist_del_first(&krcp->bkvcache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) static inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) put_cached_bnode(struct kfree_rcu_cpu *krcp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) struct kvfree_rcu_bulk_data *bnode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) // Check the limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) llist_add((struct llist_node *) bnode, &krcp->bkvcache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) krcp->nr_bkv_objs++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) * This function is invoked in workqueue context after a grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) * It frees all the objects queued on ->bhead_free or ->head_free.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) static void kfree_rcu_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) struct kvfree_rcu_bulk_data *bkvhead[FREE_N_CHANNELS], *bnext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) struct rcu_head *head, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) struct kfree_rcu_cpu *krcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) struct kfree_rcu_cpu_work *krwp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) krwp = container_of(to_rcu_work(work),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) struct kfree_rcu_cpu_work, rcu_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) krcp = krwp->krcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) raw_spin_lock_irqsave(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) // Channels 1 and 2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) for (i = 0; i < FREE_N_CHANNELS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) bkvhead[i] = krwp->bkvhead_free[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) krwp->bkvhead_free[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) // Channel 3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) head = krwp->head_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) krwp->head_free = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) raw_spin_unlock_irqrestore(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) // Handle two first channels.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) for (i = 0; i < FREE_N_CHANNELS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) for (; bkvhead[i]; bkvhead[i] = bnext) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) bnext = bkvhead[i]->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) debug_rcu_bhead_unqueue(bkvhead[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) rcu_lock_acquire(&rcu_callback_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) if (i == 0) { // kmalloc() / kfree().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) trace_rcu_invoke_kfree_bulk_callback(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) rcu_state.name, bkvhead[i]->nr_records,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) bkvhead[i]->records);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) kfree_bulk(bkvhead[i]->nr_records,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) bkvhead[i]->records);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) } else { // vmalloc() / vfree().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) for (j = 0; j < bkvhead[i]->nr_records; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) trace_rcu_invoke_kvfree_callback(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) rcu_state.name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) bkvhead[i]->records[j], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) vfree(bkvhead[i]->records[j]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) rcu_lock_release(&rcu_callback_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) raw_spin_lock_irqsave(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) if (put_cached_bnode(krcp, bkvhead[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) bkvhead[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) raw_spin_unlock_irqrestore(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) if (bkvhead[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) free_page((unsigned long) bkvhead[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) cond_resched_tasks_rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) * Emergency case only. It can happen under low memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) * condition when an allocation gets failed, so the "bulk"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) * path can not be temporary maintained.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) for (; head; head = next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) unsigned long offset = (unsigned long)head->func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) void *ptr = (void *)head - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) next = head->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) debug_rcu_head_unqueue((struct rcu_head *)ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) rcu_lock_acquire(&rcu_callback_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) if (!WARN_ON_ONCE(!__is_kvfree_rcu_offset(offset)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) kvfree(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) rcu_lock_release(&rcu_callback_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) cond_resched_tasks_rcu_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) * Schedule the kfree batch RCU work to run in workqueue context after a GP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) * This function is invoked by kfree_rcu_monitor() when the KFREE_DRAIN_JIFFIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) * timeout has been reached.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) struct kfree_rcu_cpu_work *krwp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) bool repeat = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) lockdep_assert_held(&krcp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) for (i = 0; i < KFREE_N_BATCHES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) krwp = &(krcp->krw_arr[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) * Try to detach bkvhead or head and attach it over any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) * available corresponding free channel. It can be that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) * a previous RCU batch is in progress, it means that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) * immediately to queue another one is not possible so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) * return false to tell caller to retry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) if ((krcp->bkvhead[0] && !krwp->bkvhead_free[0]) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) (krcp->bkvhead[1] && !krwp->bkvhead_free[1]) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) (krcp->head && !krwp->head_free)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) // Channel 1 corresponds to SLAB ptrs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) // Channel 2 corresponds to vmalloc ptrs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) for (j = 0; j < FREE_N_CHANNELS; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) if (!krwp->bkvhead_free[j]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) krwp->bkvhead_free[j] = krcp->bkvhead[j];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) krcp->bkvhead[j] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) // Channel 3 corresponds to emergency path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) if (!krwp->head_free) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) krwp->head_free = krcp->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) krcp->head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) WRITE_ONCE(krcp->count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) * One work is per one batch, so there are three
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) * "free channels", the batch can handle. It can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) * be that the work is in the pending state when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) * channels have been detached following by each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) * other.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) queue_rcu_work(system_wq, &krwp->rcu_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) // Repeat if any "free" corresponding channel is still busy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) if (krcp->bkvhead[0] || krcp->bkvhead[1] || krcp->head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) repeat = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) return !repeat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) // Attempt to start a new batch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) krcp->monitor_todo = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) if (queue_kfree_rcu_work(krcp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) // Success! Our job is done here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) raw_spin_unlock_irqrestore(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) // Previous RCU batch still in progress, try again later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) krcp->monitor_todo = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) raw_spin_unlock_irqrestore(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) * This function is invoked after the KFREE_DRAIN_JIFFIES timeout.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) * It invokes kfree_rcu_drain_unlock() to attempt to start another batch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) static void kfree_rcu_monitor(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) struct kfree_rcu_cpu *krcp = container_of(work, struct kfree_rcu_cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) monitor_work.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) raw_spin_lock_irqsave(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) if (krcp->monitor_todo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) kfree_rcu_drain_unlock(krcp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) raw_spin_unlock_irqrestore(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) static enum hrtimer_restart
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) schedule_page_work_fn(struct hrtimer *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) struct kfree_rcu_cpu *krcp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) container_of(t, struct kfree_rcu_cpu, hrtimer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) queue_work(system_highpri_wq, &krcp->page_cache_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) return HRTIMER_NORESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) static void fill_page_cache_func(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) struct kvfree_rcu_bulk_data *bnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) struct kfree_rcu_cpu *krcp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) container_of(work, struct kfree_rcu_cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) page_cache_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) bool pushed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) for (i = 0; i < rcu_min_cached_objs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) bnode = (struct kvfree_rcu_bulk_data *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390) __get_free_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) if (bnode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) raw_spin_lock_irqsave(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) pushed = put_cached_bnode(krcp, bnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) raw_spin_unlock_irqrestore(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) if (!pushed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) free_page((unsigned long) bnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) atomic_set(&krcp->work_in_progress, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) run_page_cache_worker(struct kfree_rcu_cpu *krcp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) !atomic_xchg(&krcp->work_in_progress, 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) krcp->hrtimer.function = schedule_page_work_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) static inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) struct kvfree_rcu_bulk_data *bnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) if (unlikely(!krcp->initialized))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) lockdep_assert_held(&krcp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) idx = !!is_vmalloc_addr(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) /* Check if a new block is required. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) if (!krcp->bkvhead[idx] ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) bnode = get_cached_bnode(krcp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) /* Switch to emergency path. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) if (!bnode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) /* Initialize the new block. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) bnode->nr_records = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) bnode->next = krcp->bkvhead[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) /* Attach it to the head. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) krcp->bkvhead[idx] = bnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) /* Finally insert. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) krcp->bkvhead[idx]->records
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) [krcp->bkvhead[idx]->nr_records++] = ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) * Queue a request for lazy invocation of appropriate free routine after a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) * grace period. Please note there are three paths are maintained, two are the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) * main ones that use array of pointers interface and third one is emergency
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) * one, that is used only when the main path can not be maintained temporary,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) * due to memory pressure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) * Each kvfree_call_rcu() request is added to a batch. The batch will be drained
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) * be free'd in workqueue context. This allows us to: batch requests together to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469) struct kfree_rcu_cpu *krcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) bool success;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) void *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) if (head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) ptr = (void *) head - (unsigned long) func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) * Please note there is a limitation for the head-less
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) * variant, that is why there is a clear rule for such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) * objects: it can be used from might_sleep() context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) * only. For other places please embed an rcu_head to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) * your data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) ptr = (unsigned long *) func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) krcp = krc_this_cpu_lock(&flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) // Queue the object but don't yet schedule the batch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) if (debug_rcu_head_queue(ptr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) // Probable double kfree_rcu(), just leak.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) __func__, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) // Mark as success and leave.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) success = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) goto unlock_return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501) if (!success) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) run_page_cache_worker(krcp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) if (head == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) // Inline if kvfree_rcu(one_arg) call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) goto unlock_return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) head->func = func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) head->next = krcp->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) krcp->head = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) success = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) WRITE_ONCE(krcp->count, krcp->count + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) // Set timer to drain after KFREE_DRAIN_JIFFIES.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518) !krcp->monitor_todo) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) krcp->monitor_todo = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) unlock_return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) krc_this_cpu_unlock(krcp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) * Inline kvfree() after synchronize_rcu(). We can do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528) * it from might_sleep() context only, so the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529) * CPU can pass the QS state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) if (!success) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532) debug_rcu_head_unqueue((struct rcu_head *) ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) kvfree(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) EXPORT_SYMBOL_GPL(kvfree_call_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) static unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540) kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) unsigned long count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) /* Snapshot count of all CPUs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) count += READ_ONCE(krcp->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) static unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) kfree_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) int cpu, freed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) count = krcp->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) raw_spin_lock_irqsave(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567) if (krcp->monitor_todo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) kfree_rcu_drain_unlock(krcp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570) raw_spin_unlock_irqrestore(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572) sc->nr_to_scan -= count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) freed += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) if (sc->nr_to_scan <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579) return freed == 0 ? SHRINK_STOP : freed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) static struct shrinker kfree_rcu_shrinker = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) .count_objects = kfree_rcu_shrink_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) .scan_objects = kfree_rcu_shrink_scan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) .batch = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) .seeks = DEFAULT_SEEKS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) void __init kfree_rcu_scheduler_running(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) raw_spin_lock_irqsave(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598) if (!krcp->head || krcp->monitor_todo) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) raw_spin_unlock_irqrestore(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) krcp->monitor_todo = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) schedule_delayed_work_on(cpu, &krcp->monitor_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) KFREE_DRAIN_JIFFIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) raw_spin_unlock_irqrestore(&krcp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) * During early boot, any blocking grace-period wait automatically
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) * implies a grace period. Later on, this is never the case for PREEMPTION.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) * Howevr, because a context switch is a grace period for !PREEMPTION, any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) * blocking grace-period wait automatically implies a grace period if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) * there is only one CPU online at any point time during execution of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) * occasionally incorrectly indicate that there are multiple CPUs online
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618) * when there was in fact only one the whole time, as this just adds some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) * overhead: RCU still operates correctly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) static int rcu_blocking_is_gp(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) if (IS_ENABLED(CONFIG_PREEMPTION))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) might_sleep(); /* Check for RCU read-side critical section. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) ret = num_online_cpus() <= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) * synchronize_rcu - wait until a grace period has elapsed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637) * Control will return to the caller some time after a full grace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) * period has elapsed, in other words after all currently executing RCU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) * read-side critical sections have completed. Note, however, that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640) * upon return from synchronize_rcu(), the caller might well be executing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) * concurrently with new RCU read-side critical sections that began while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) * synchronize_rcu() was waiting. RCU read-side critical sections are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) * In addition, regions of code across which interrupts, preemption, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) * softirqs have been disabled also serve as RCU read-side critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) * sections. This includes hardware interrupt handlers, softirq handlers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647) * and NMI handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) * Note that this guarantee implies further memory-ordering guarantees.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) * On systems with more than one CPU, when synchronize_rcu() returns,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) * each CPU is guaranteed to have executed a full memory barrier since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) * the end of its last RCU read-side critical section whose beginning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) * preceded the call to synchronize_rcu(). In addition, each CPU having
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) * an RCU read-side critical section that extends beyond the return from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) * synchronize_rcu() is guaranteed to have executed a full memory barrier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) * after the beginning of synchronize_rcu() and before the beginning of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) * that RCU read-side critical section. Note that these guarantees include
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) * CPUs that are offline, idle, or executing in user mode, as well as CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) * that are executing in the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661) * Furthermore, if CPU A invoked synchronize_rcu(), which returned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) * to its caller on CPU B, then both CPU A and CPU B are guaranteed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) * to have executed a full memory barrier during the execution of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) * synchronize_rcu() -- even if CPU A and CPU B are the same CPU (but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) * again only if the system has more than one CPU).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) void synchronize_rcu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) lock_is_held(&rcu_lock_map) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) lock_is_held(&rcu_sched_lock_map),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) "Illegal synchronize_rcu() in RCU read-side critical section");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) if (rcu_blocking_is_gp())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) if (rcu_gp_is_expedited())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) synchronize_rcu_expedited();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) wait_rcu_gp(call_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) EXPORT_SYMBOL_GPL(synchronize_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) * get_state_synchronize_rcu - Snapshot current RCU state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) * Returns a cookie that is used by a later call to cond_synchronize_rcu()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) * to determine whether or not a full grace period has elapsed in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687) * meantime.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) unsigned long get_state_synchronize_rcu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) * Any prior manipulation of RCU-protected data must happen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) * before the load from ->gp_seq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) smp_mb(); /* ^^^ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) return rcu_seq_snap(&rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) * cond_synchronize_rcu - Conditionally wait for an RCU grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) * @oldstate: return value from earlier call to get_state_synchronize_rcu()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) * If a full RCU grace period has elapsed since the earlier call to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) * get_state_synchronize_rcu(), just return. Otherwise, invoke
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) * synchronize_rcu() to wait for a full grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) * Yes, this function does not take counter wrap into account. But
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) * counter wrap is harmless. If the counter wraps, we have waited for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) * more than 2 billion grace periods (and way more on a 64-bit system!),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) * so waiting for one additional grace period should be just fine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714) void cond_synchronize_rcu(unsigned long oldstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) smp_mb(); /* Ensure GP ends before subsequent accesses. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) EXPORT_SYMBOL_GPL(cond_synchronize_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) * Check to see if there is any immediate RCU-related work to be done by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725) * the current CPU, returning 1 if so and zero otherwise. The checks are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) * in order of increasing expense: checks that can be carried out against
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727) * CPU-local state are performed first. However, we must check for CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) * stalls first, else we might not get a chance.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) static int rcu_pending(int user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) bool gp_in_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) struct rcu_node *rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) /* Check for CPU stalls, if enabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) check_cpu_stall(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) /* Does this CPU need a deferred NOCB wakeup? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) if (rcu_nocb_need_deferred_wakeup(rdp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) /* Is this a nohz_full CPU in userspace or idle? (Ignore RCU if so.) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) if ((user || rcu_is_cpu_rrupt_from_idle()) && rcu_nohz_full_cpu())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749) /* Is the RCU core waiting for a quiescent state from this CPU? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) gp_in_progress = rcu_gp_in_progress();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm && gp_in_progress)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) /* Does this CPU have callbacks ready to invoke? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755) if (rcu_segcblist_ready_cbs(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) /* Has RCU gone idle with this CPU needing another grace period? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760) (!IS_ENABLED(CONFIG_RCU_NOCB_CPU) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) !rcu_segcblist_is_offloaded(&rdp->cblist)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) /* Have RCU grace period completed or started? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766) if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) /* nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) * Helper function for rcu_barrier() tracing. If tracing is disabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) * the compiler is expected to optimize this away.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) static void rcu_barrier_trace(const char *s, int cpu, unsigned long done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780) trace_rcu_barrier(rcu_state.name, s, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781) atomic_read(&rcu_state.barrier_cpu_count), done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) * RCU callback function for rcu_barrier(). If we are last, wake
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) * up the task executing rcu_barrier().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788) * Note that the value of rcu_state.barrier_sequence must be captured
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) * before the atomic_dec_and_test(). Otherwise, if this CPU is not last,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790) * other CPUs might count the value down to zero before this CPU gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) * around to invoking rcu_barrier_trace(), which might result in bogus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) * data from the next instance of rcu_barrier().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) static void rcu_barrier_callback(struct rcu_head *rhp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796) unsigned long __maybe_unused s = rcu_state.barrier_sequence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) rcu_barrier_trace(TPS("LastCB"), -1, s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) complete(&rcu_state.barrier_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) rcu_barrier_trace(TPS("CB"), -1, s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) * Called with preemption disabled, and from cross-cpu IRQ context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809) static void rcu_barrier_func(void *cpu_in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) uintptr_t cpu = (uintptr_t)cpu_in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) rdp->barrier_head.func = rcu_barrier_callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816) debug_rcu_head_queue(&rdp->barrier_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) rcu_nocb_lock(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820) atomic_inc(&rcu_state.barrier_cpu_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) debug_rcu_head_unqueue(&rdp->barrier_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) rcu_barrier_trace(TPS("IRQNQ"), -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) rcu_nocb_unlock(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) * Note that this primitive does not necessarily wait for an RCU grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) * to complete. For example, if there are no RCU callbacks queued anywhere
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) * in the system, then rcu_barrier() is within its rights to return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) * immediately, without waiting for anything, much less an RCU grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) void rcu_barrier(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) uintptr_t cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843) rcu_barrier_trace(TPS("Begin"), -1, s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) /* Take mutex to serialize concurrent rcu_barrier() requests. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) mutex_lock(&rcu_state.barrier_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) /* Did someone else do our work for us? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) rcu_barrier_trace(TPS("EarlyExit"), -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851) rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) smp_mb(); /* caller's subsequent code after above check. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) mutex_unlock(&rcu_state.barrier_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857) /* Mark the start of the barrier operation. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) rcu_seq_start(&rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859) rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862) * Initialize the count to two rather than to zero in order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) * to avoid a too-soon return to zero in case of an immediate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) * invocation of the just-enqueued callback (or preemption of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) * this task). Exclude CPU-hotplug operations to ensure that no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) * offline non-offloaded CPU has callbacks queued.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) init_completion(&rcu_state.barrier_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) atomic_set(&rcu_state.barrier_cpu_count, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) get_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) * Force each CPU with callbacks to register a new callback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) * When that callback is invoked, we will know that all of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) * corresponding CPU's preceding callbacks have been invoked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) if (cpu_is_offline(cpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) !rcu_segcblist_is_offloaded(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) rcu_barrier_trace(TPS("OnlineQ"), cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884) rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885) smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) cpu_is_offline(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) rcu_barrier_func((void *)cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) } else if (cpu_is_offline(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895) rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) rcu_barrier_trace(TPS("OnlineNQ"), cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901) put_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904) * Now that we have an rcu_barrier_callback() callback on each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) * CPU, and thus each counted, remove the initial count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908) complete(&rcu_state.barrier_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) /* Wait for all rcu_barrier_callback() callbacks to be invoked. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) wait_for_completion(&rcu_state.barrier_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) /* Mark the end of the barrier operation. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) rcu_seq_end(&rcu_state.barrier_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) /* Other rcu_barrier() invocations can now safely proceed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) mutex_unlock(&rcu_state.barrier_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920) EXPORT_SYMBOL_GPL(rcu_barrier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923) * Propagate ->qsinitmask bits up the rcu_node tree to account for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) * first CPU in a given leaf rcu_node structure coming online. The caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) * must hold the corresponding leaf rcu_node ->lock with interrrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926) * disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) static void rcu_init_new_rnp(struct rcu_node *rnp_leaf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930) long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) long oldmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) struct rcu_node *rnp = rnp_leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) raw_lockdep_assert_held_rcu_node(rnp_leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) WARN_ON_ONCE(rnp->wait_blkd_tasks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937) mask = rnp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) rnp = rnp->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) if (rnp == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) raw_spin_lock_rcu_node(rnp); /* Interrupts already disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) oldmask = rnp->qsmaskinit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) rnp->qsmaskinit |= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) raw_spin_unlock_rcu_node(rnp); /* Interrupts remain disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) if (oldmask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) * Do boot-time initialization of a CPU's per-CPU RCU data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) static void __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) rcu_boot_init_percpu_data(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) /* Set up local state, ensuring consistent view of global state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959) rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) INIT_WORK(&rdp->strict_work, strict_work_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) WARN_ON_ONCE(rdp->dynticks_nesting != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) rdp->rcu_onl_gp_seq = rcu_state.gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) rdp->rcu_onl_gp_flags = RCU_GP_CLEANED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) rdp->cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968) rcu_boot_init_nocb_percpu_data(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972) * Invoked early in the CPU-online process, when pretty much all services
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) * are available. The incoming CPU is not present.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) * Initializes a CPU's per-CPU RCU data. Note that only one online or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) * offline event can be happening at a given time. Note also that we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) * accept some slop in the rsp->gp_seq access due to the fact that this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) * CPU cannot possibly have any non-offloaded RCU callbacks in flight yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) * And any offloaded callbacks are being numbered elsewhere.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) int rcutree_prepare_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) struct rcu_node *rnp = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) /* Set up local state, ensuring consistent view of global state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) rdp->qlen_last_fqs_check = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) rdp->blimit = blimit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) !rcu_segcblist_is_offloaded(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996) rcu_dynticks_eqs_online();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997) raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000) * Add CPU to leaf rcu_node pending-online bitmask. Any needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) * propagation up the rcu_node tree will happen at the beginning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) * of the next grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005) raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) rdp->beenonline = true; /* We have now been online. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) rdp->gp_seq = READ_ONCE(rnp->gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) rdp->gp_seq_needed = rdp->gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) rdp->cpu_no_qs.b.norm = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010) rdp->core_needs_qs = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) rdp->rcu_iw_pending = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015) rcu_prepare_kthreads(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) rcu_spawn_cpu_nocb_kthread(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) * Update RCU priority boot kthread affinity for CPU-hotplug changes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024) static void rcutree_affinity_setting(unsigned int cpu, int outgoing)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) rcu_boost_kthread_setaffinity(rdp->mynode, outgoing);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) * Near the end of the CPU-online process. Pretty much all services
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) * enabled, and the CPU is now very much alive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) int rcutree_online_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041) rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) rnp->ffmask |= rdp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) return 0; /* Too early in boot for scheduler work. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) sync_sched_exp_online_cleanup(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) rcutree_affinity_setting(cpu, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051) // Stop-machine done, so allow nohz_full to disable tick.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) tick_dep_clear(TICK_DEP_BIT_RCU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) * Near the beginning of the process. The CPU is still very much alive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) * with pretty much all services enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) int rcutree_offline_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067) rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) rnp->ffmask &= ~rdp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) rcutree_affinity_setting(cpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) // nohz_full CPUs need the tick for stop-machine to work quickly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) tick_dep_set(TICK_DEP_BIT_RCU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) * Mark the specified CPU as being online so that subsequent grace periods
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) * (both expedited and normal) will wait on it. Note that this means that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082) * incoming CPUs are not allowed to use RCU read-side critical sections
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083) * until this function is called. Failing to observe this restriction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084) * will result in lockdep splats.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) * Note that this function is special in that it is invoked directly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087) * from the incoming CPU rather than from the cpuhp_step mechanism.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) * This is because this function must be invoked at a precise location.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) void rcu_cpu_starting(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) struct rcu_data *rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) bool newcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) if (rdp->cpu_started)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101) rdp->cpu_started = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) rnp = rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) mask = rdp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext | mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) newcpu = !(rnp->expmaskinitnext & mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) rnp->expmaskinitnext |= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) /* Allow lockless access for expedited grace periods. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113) rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116) rcu_disable_urgency_upon_qs(rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) /* Report QS -after- changing ->qsmaskinitnext! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118) rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) * The outgoing function has no further need of RCU, so remove it from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127) * the rcu_node tree's ->qsmaskinitnext bit masks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) * Note that this function is special in that it is invoked directly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) * from the outgoing CPU rather than from the cpuhp_step mechanism.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) * This is because this function must be invoked at a precise location.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) void rcu_report_dead(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) /* QS for any half-done expedited grace period. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142) rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) rcu_preempt_deferred_qs(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) mask = rdp->grpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) raw_spin_lock(&rcu_state.ofl_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151) rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) /* Report quiescent state -before- changing ->qsmaskinitnext! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154) rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) raw_spin_unlock(&rcu_state.ofl_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) rdp->cpu_started = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166) * The outgoing CPU has just passed through the dying-idle state, and we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167) * are being invoked from the CPU that was IPIed to continue the offline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168) * operation. Migrate the outgoing CPU's callbacks to the current CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) void rcutree_migrate_callbacks(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) struct rcu_data *my_rdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174) struct rcu_node *my_rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) bool needwake;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) if (rcu_segcblist_is_offloaded(&rdp->cblist) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179) rcu_segcblist_empty(&rdp->cblist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180) return; /* No callbacks to migrate. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) my_rdp = this_cpu_ptr(&rcu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184) my_rnp = my_rdp->mynode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) rcu_nocb_lock(my_rdp); /* irqs already disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186) WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187) raw_spin_lock_rcu_node(my_rnp); /* irqs already disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188) /* Leverage recent GPs and set GP for new callbacks. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) needwake = rcu_advance_cbs(my_rnp, rdp) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) rcu_advance_cbs(my_rnp, my_rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) needwake = needwake || rcu_advance_cbs(my_rnp, my_rdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193) rcu_segcblist_disable(&rdp->cblist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194) WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) !rcu_segcblist_n_cbs(&my_rdp->cblist));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) raw_spin_unlock_rcu_node(my_rnp); /* irqs remain disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) __call_rcu_nocb_wake(my_rdp, true, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200) rcu_nocb_unlock(my_rdp); /* irqs remain disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201) raw_spin_unlock_irqrestore_rcu_node(my_rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203) if (needwake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204) rcu_gp_kthread_wake();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205) lockdep_assert_irqs_enabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207) !rcu_segcblist_empty(&rdp->cblist),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209) cpu, rcu_segcblist_n_cbs(&rdp->cblist),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) rcu_segcblist_first_cb(&rdp->cblist));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215) * On non-huge systems, use expedited RCU grace periods to make suspend
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216) * and hibernation run faster.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) static int rcu_pm_notify(struct notifier_block *self,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) unsigned long action, void *hcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) switch (action) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222) case PM_HIBERNATION_PREPARE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223) case PM_SUSPEND_PREPARE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) rcu_expedite_gp();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) case PM_POST_HIBERNATION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) case PM_POST_SUSPEND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) rcu_unexpedite_gp();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237) * Spawn the kthreads that handle RCU's grace periods.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239) static int __init rcu_spawn_gp_kthread(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242) int kthread_prio_in = kthread_prio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) struct sched_param sp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245) struct task_struct *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) /* Force priority into range. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248) if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249) && IS_BUILTIN(CONFIG_RCU_TORTURE_TEST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250) kthread_prio = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) else if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252) kthread_prio = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) else if (kthread_prio < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254) kthread_prio = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) else if (kthread_prio > 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256) kthread_prio = 99;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258) if (kthread_prio != kthread_prio_in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) pr_alert("rcu_spawn_gp_kthread(): Limited prio to %d from %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260) kthread_prio, kthread_prio_in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) rcu_scheduler_fully_active = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263) t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) if (WARN_ONCE(IS_ERR(t), "%s: Could not start grace-period kthread, OOM is now expected behavior\n", __func__))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266) if (kthread_prio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267) sp.sched_priority = kthread_prio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270) rnp = rcu_get_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) raw_spin_lock_irqsave_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272) WRITE_ONCE(rcu_state.gp_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274) // Reset .gp_activity and .gp_req_activity before setting .gp_kthread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275) smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276) raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) wake_up_process(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278) rcu_spawn_nocb_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) rcu_spawn_boost_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280) rcu_spawn_core_kthreads();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) early_initcall(rcu_spawn_gp_kthread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286) * This function is invoked towards the end of the scheduler's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) * initialization process. Before this is called, the idle task might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) * contain synchronous grace-period primitives (during which time, this idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) * task is booting the system, and such primitives are no-ops). After this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290) * function is called, any synchronous grace-period primitives are run as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) * expedited, with the requesting task driving the grace period forward.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292) * A later core_initcall() rcu_set_runtime_mode() will switch to full
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) * runtime RCU functionality.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) void rcu_scheduler_starting(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297) WARN_ON(num_online_cpus() != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298) WARN_ON(nr_context_switches() > 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299) rcu_test_sync_prims();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) rcu_scheduler_active = RCU_SCHEDULER_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301) rcu_test_sync_prims();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305) * Helper function for rcu_init() that initializes the rcu_state structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) static void __init rcu_init_one(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) static const char * const buf[] = RCU_NODE_NAME_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) static const char * const fqs[] = RCU_FQS_NAME_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314) int levelspread[RCU_NUM_LVLS]; /* kids/node in each level. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) int cpustride = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) int j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320) BUILD_BUG_ON(RCU_NUM_LVLS > ARRAY_SIZE(buf)); /* Fix buf[] init! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322) /* Silence gcc 4.8 false positive about array index out of range. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323) if (rcu_num_lvls <= 0 || rcu_num_lvls > RCU_NUM_LVLS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324) panic("rcu_init_one: rcu_num_lvls out of range");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) /* Initialize the level-tracking arrays. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328) for (i = 1; i < rcu_num_lvls; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) rcu_state.level[i] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330) rcu_state.level[i - 1] + num_rcu_lvl[i - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331) rcu_init_levelspread(levelspread, num_rcu_lvl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333) /* Initialize the elements themselves, starting from the leaves. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335) for (i = rcu_num_lvls - 1; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) cpustride *= levelspread[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) rnp = rcu_state.level[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338) for (j = 0; j < num_rcu_lvl[i]; j++, rnp++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) raw_spin_lock_init(&ACCESS_PRIVATE(rnp, lock));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp, lock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341) &rcu_node_class[i], buf[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342) raw_spin_lock_init(&rnp->fqslock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) lockdep_set_class_and_name(&rnp->fqslock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344) &rcu_fqs_class[i], fqs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) rnp->gp_seq = rcu_state.gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346) rnp->gp_seq_needed = rcu_state.gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347) rnp->completedqs = rcu_state.gp_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348) rnp->qsmask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) rnp->qsmaskinit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) rnp->grplo = j * cpustride;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351) rnp->grphi = (j + 1) * cpustride - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352) if (rnp->grphi >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) rnp->grphi = nr_cpu_ids - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) if (i == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355) rnp->grpnum = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) rnp->grpmask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357) rnp->parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) rnp->grpnum = j % levelspread[i - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) rnp->grpmask = BIT(rnp->grpnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361) rnp->parent = rcu_state.level[i - 1] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362) j / levelspread[i - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) rnp->level = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365) INIT_LIST_HEAD(&rnp->blkd_tasks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366) rcu_init_one_nocb(rnp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) init_waitqueue_head(&rnp->exp_wq[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368) init_waitqueue_head(&rnp->exp_wq[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) init_waitqueue_head(&rnp->exp_wq[2]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370) init_waitqueue_head(&rnp->exp_wq[3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371) spin_lock_init(&rnp->exp_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) init_swait_queue_head(&rcu_state.gp_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376) init_swait_queue_head(&rcu_state.expedited_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377) rnp = rcu_first_leaf_node();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) for_each_possible_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) while (i > rnp->grphi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) rnp++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381) per_cpu_ptr(&rcu_data, i)->mynode = rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) rcu_boot_init_percpu_data(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) * Compute the rcu_node tree geometry from kernel parameters. This cannot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388) * replace the definitions in tree.h because those are needed to size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389) * the ->node array in the rcu_state structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) void rcu_init_geometry(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393) ulong d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395) static unsigned long old_nr_cpu_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) int rcu_capacity[RCU_NUM_LVLS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) static bool initialized;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399) if (initialized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401) * Warn if setup_nr_cpu_ids() had not yet been invoked,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) * unless nr_cpus_ids == NR_CPUS, in which case who cares?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404) WARN_ON_ONCE(old_nr_cpu_ids != nr_cpu_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) old_nr_cpu_ids = nr_cpu_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412) * Initialize any unspecified boot parameters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) * The default values of jiffies_till_first_fqs and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) * jiffies_till_next_fqs are set to the RCU_JIFFIES_TILL_FORCE_QS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) * value, which is a function of HZ, then adding one for each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416) * RCU_JIFFIES_FQS_DIV CPUs that might be on the system.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418) d = RCU_JIFFIES_TILL_FORCE_QS + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) if (jiffies_till_first_fqs == ULONG_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420) jiffies_till_first_fqs = d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) if (jiffies_till_next_fqs == ULONG_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422) jiffies_till_next_fqs = d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423) adjust_jiffies_till_sched_qs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425) /* If the compile-time values are accurate, just leave. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427) nr_cpu_ids == NR_CPUS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) pr_info("Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430) rcu_fanout_leaf, nr_cpu_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) * The boot-time rcu_fanout_leaf parameter must be at least two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434) * and cannot exceed the number of bits in the rcu_node masks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435) * Complain and fall back to the compile-time values if this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) * limit is exceeded.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) if (rcu_fanout_leaf < 2 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439) rcu_fanout_leaf > sizeof(unsigned long) * 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) rcu_fanout_leaf = RCU_FANOUT_LEAF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446) * Compute number of nodes that can be handled an rcu_node tree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) * with the given number of levels.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449) rcu_capacity[0] = rcu_fanout_leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450) for (i = 1; i < RCU_NUM_LVLS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451) rcu_capacity[i] = rcu_capacity[i - 1] * RCU_FANOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) * The tree must be able to accommodate the configured number of CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) * If this limit is exceeded, fall back to the compile-time values.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) if (nr_cpu_ids > rcu_capacity[RCU_NUM_LVLS - 1]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) rcu_fanout_leaf = RCU_FANOUT_LEAF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) /* Calculate the number of levels in the tree. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464) for (i = 0; nr_cpu_ids > rcu_capacity[i]; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) rcu_num_lvls = i + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468) /* Calculate the number of rcu_nodes at each level of the tree. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) for (i = 0; i < rcu_num_lvls; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) int cap = rcu_capacity[(rcu_num_lvls - 1) - i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) num_rcu_lvl[i] = DIV_ROUND_UP(nr_cpu_ids, cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474) /* Calculate the total number of rcu_node structures. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) rcu_num_nodes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476) for (i = 0; i < rcu_num_lvls; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477) rcu_num_nodes += num_rcu_lvl[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) * Dump out the structure of the rcu_node combining tree associated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482) * with the rcu_state structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484) static void __init rcu_dump_rcu_node_tree(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486) int level = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487) struct rcu_node *rnp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) pr_info("rcu_node tree layout dump\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490) pr_info(" ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491) rcu_for_each_node_breadth_first(rnp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492) if (rnp->level != level) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494) pr_info(" ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495) level = rnp->level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497) pr_cont("%d:%d ^%d ", rnp->grplo, rnp->grphi, rnp->grpnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) struct workqueue_struct *rcu_gp_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) struct workqueue_struct *rcu_par_gp_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) static void __init kfree_rcu_batch_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511) struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) for (i = 0; i < KFREE_N_BATCHES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514) INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515) krcp->krw_arr[i].krcp = krcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519) INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) krcp->initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) if (register_shrinker(&kfree_rcu_shrinker))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523) pr_err("Failed to register kfree_rcu() shrinker!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526) void __init rcu_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530) rcu_early_boot_tests();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532) kfree_rcu_batch_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) rcu_bootup_announce();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534) rcu_init_geometry();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535) rcu_init_one();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536) if (dump_tree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537) rcu_dump_rcu_node_tree();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538) if (use_softirq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539) open_softirq(RCU_SOFTIRQ, rcu_core_si);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542) * We don't need protection against CPU-hotplug here because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543) * this is called early in boot, before either interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544) * or the scheduler are operational.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546) pm_notifier(rcu_pm_notify, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548) rcutree_prepare_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549) rcu_cpu_starting(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550) rcutree_online_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553) /* Create workqueue for expedited GPs and for Tree SRCU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554) rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555) WARN_ON(!rcu_gp_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556) rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) WARN_ON(!rcu_par_gp_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558) srcu_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560) /* Fill in default value for rcutree.qovld boot parameter. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561) /* -After- the rcu_node ->lock fields are initialized! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562) if (qovld < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563) qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565) qovld_calc = qovld;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568) #include "tree_stall.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) #include "tree_exp.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570) #include "tree_plugin.h"
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags