^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * No idle tick implementation for low and high resolution timers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Started by: Thomas Gleixner and Ingo Molnar
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/hrtimer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/kernel_stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/profile.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/sched/clock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/sched/stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/sched/nohz.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/irq_work.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/posix-timers.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/context_tracking.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <trace/hooks/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/irq_regs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include "tick-internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <trace/events/timer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * Per-CPU nohz control structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) struct tick_sched *tick_get_tick_sched(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return &per_cpu(tick_cpu_sched, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * The time, when the last jiffy update happened. Protected by jiffies_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static ktime_t last_jiffies_update;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * Must be called with interrupts disabled !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static void tick_do_update_jiffies64(ktime_t now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) unsigned long ticks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) ktime_t delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * Do a quick check without holding jiffies_lock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * The READ_ONCE() pairs with two updates done later in this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) delta = ktime_sub(now, READ_ONCE(last_jiffies_update));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) if (delta < tick_period)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* Reevaluate with jiffies_lock held */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) raw_spin_lock(&jiffies_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) write_seqcount_begin(&jiffies_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) delta = ktime_sub(now, last_jiffies_update);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) if (delta >= tick_period) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) delta = ktime_sub(delta, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /* Pairs with the lockless read in this function. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) WRITE_ONCE(last_jiffies_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) ktime_add(last_jiffies_update, tick_period));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /* Slow path for long timeouts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) if (unlikely(delta >= tick_period)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) s64 incr = ktime_to_ns(tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) ticks = ktime_divns(delta, incr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) /* Pairs with the lockless read in this function. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) WRITE_ONCE(last_jiffies_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) ktime_add_ns(last_jiffies_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) incr * ticks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) do_timer(++ticks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) /* Keep the tick_next_period variable up to date */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) tick_next_period = ktime_add(last_jiffies_update, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) write_seqcount_end(&jiffies_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) raw_spin_unlock(&jiffies_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) write_seqcount_end(&jiffies_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) raw_spin_unlock(&jiffies_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) update_wall_time();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * Initialize and return retrieve the jiffies update.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static ktime_t tick_init_jiffy_update(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) ktime_t period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) raw_spin_lock(&jiffies_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) write_seqcount_begin(&jiffies_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /* Did we start the jiffies update yet ? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (last_jiffies_update == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) last_jiffies_update = tick_next_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) period = last_jiffies_update;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) write_seqcount_end(&jiffies_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) raw_spin_unlock(&jiffies_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) return period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #ifdef CONFIG_NO_HZ_COMMON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * Check if the do_timer duty was dropped. We don't care about
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * concurrency: This happens only when the CPU in charge went
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * into a long sleep. If two CPUs happen to assign themselves to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * this duty, then the jiffies update is still serialized by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * jiffies_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * If nohz_full is enabled, this should not happen because the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * tick_do_timer_cpu never relinquishes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #ifdef CONFIG_NO_HZ_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) WARN_ON(tick_nohz_full_running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) tick_do_timer_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) /* Check, if the jiffies need an update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) if (tick_do_timer_cpu == cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) tick_do_update_jiffies64(now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) trace_android_vh_jiffies_update(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (ts->inidle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) ts->got_idle_tick = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #ifdef CONFIG_NO_HZ_COMMON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * When we are idle and the tick is stopped, we have to touch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * the watchdog as we might not schedule for a really long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * time. This happens on complete idle SMP systems while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * waiting on the login prompt. We also increment the "start of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * idle" jiffy stamp so the idle accounting adjustment we do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * when we go busy again does not account too much ticks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (ts->tick_stopped) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) touch_softlockup_watchdog_sched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (is_idle_task(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) ts->idle_jiffies++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * In case the current tick fired too early past its expected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * expiration, make sure we don't bypass the next clock reprogramming
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * to the same deadline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) ts->next_tick = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) update_process_times(user_mode(regs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) profile_tick(CPU_PROFILING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #ifdef CONFIG_NO_HZ_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) cpumask_var_t tick_nohz_full_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) bool tick_nohz_full_running;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) EXPORT_SYMBOL_GPL(tick_nohz_full_running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static atomic_t tick_dep_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) static bool check_tick_dependency(atomic_t *dep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) int val = atomic_read(dep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) if (val & TICK_DEP_MASK_POSIX_TIMER) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) if (val & TICK_DEP_MASK_PERF_EVENTS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (val & TICK_DEP_MASK_SCHED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) trace_tick_stop(0, TICK_DEP_MASK_SCHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (val & TICK_DEP_MASK_RCU) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) trace_tick_stop(0, TICK_DEP_MASK_RCU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (unlikely(!cpu_online(cpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) if (check_tick_dependency(&tick_dep_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (check_tick_dependency(&ts->tick_dep_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (check_tick_dependency(¤t->tick_dep_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (check_tick_dependency(¤t->signal->tick_dep_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static void nohz_full_kick_func(struct irq_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /* Empty, the tick restart happens on tick_nohz_irq_exit() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) .func = nohz_full_kick_func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) .flags = ATOMIC_INIT(IRQ_WORK_HARD_IRQ),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * Kick this CPU if it's full dynticks in order to force it to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * re-evaluate its dependency on the tick and restart it if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * is NMI safe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static void tick_nohz_full_kick(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (!tick_nohz_full_cpu(smp_processor_id()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * Kick the CPU if it's full dynticks in order to force it to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * re-evaluate its dependency on the tick and restart it if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) void tick_nohz_full_kick_cpu(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (!tick_nohz_full_cpu(cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * Kick all full dynticks CPUs in order to force these to re-evaluate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * their dependency on the tick and restart it if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static void tick_nohz_full_kick_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (!tick_nohz_full_running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) tick_nohz_full_kick_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) static void tick_nohz_dep_set_all(atomic_t *dep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) enum tick_dep_bits bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) int prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) prev = atomic_fetch_or(BIT(bit), dep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (!prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) tick_nohz_full_kick_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * Set a global tick dependency. Used by perf events that rely on freq and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * by unstable clock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) void tick_nohz_dep_set(enum tick_dep_bits bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) tick_nohz_dep_set_all(&tick_dep_mask, bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) void tick_nohz_dep_clear(enum tick_dep_bits bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) atomic_andnot(BIT(bit), &tick_dep_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * Set per-CPU tick dependency. Used by scheduler and perf events in order to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * manage events throttling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) int prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) struct tick_sched *ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) ts = per_cpu_ptr(&tick_cpu_sched, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (!prev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) /* Perf needs local kick that is NMI safe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (cpu == smp_processor_id()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) tick_nohz_full_kick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) /* Remote irq work not NMI-safe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (!WARN_ON_ONCE(in_nmi()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) tick_nohz_full_kick_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) EXPORT_SYMBOL_GPL(tick_nohz_dep_set_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) atomic_andnot(BIT(bit), &ts->tick_dep_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * Set a per-task tick dependency. RCU need this. Also posix CPU timers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * in order to elapse per task timers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) if (tsk == current) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) tick_nohz_full_kick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * Some future tick_nohz_full_kick_task()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * should optimize this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) tick_nohz_full_kick_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * per process timers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) tick_nohz_dep_set_all(&sig->tick_dep_mask, bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) atomic_andnot(BIT(bit), &sig->tick_dep_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * Re-evaluate the need for the tick as we switch the current task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * It might need the tick due to per task/process properties:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * perf events, posix CPU timers, ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) void __tick_nohz_task_switch(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) struct tick_sched *ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (!tick_nohz_full_cpu(smp_processor_id()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (ts->tick_stopped) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (atomic_read(¤t->tick_dep_mask) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) atomic_read(¤t->signal->tick_dep_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) tick_nohz_full_kick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /* Get the boot-time nohz CPU list from the kernel parameters. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) void __init tick_nohz_full_setup(cpumask_var_t cpumask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) cpumask_copy(tick_nohz_full_mask, cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) tick_nohz_full_running = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) EXPORT_SYMBOL_GPL(tick_nohz_full_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) static int tick_nohz_cpu_down(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * The tick_do_timer_cpu CPU handles housekeeping duty (unbound
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * timers, workqueues, timekeeping, ...) on behalf of full dynticks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) * CPUs. It must remain online when nohz full is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) void __init tick_nohz_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) int cpu, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (!tick_nohz_full_running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) * Full dynticks uses irq work to drive the tick rescheduling on safe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * locking contexts. But then we need irq work to raise its own
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * interrupts to avoid circular dependency on the tick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) if (!arch_irq_work_has_interrupt()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support irq work self-IPIs\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) cpumask_clear(tick_nohz_full_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) tick_nohz_full_running = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) !IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) pr_warn("NO_HZ: Clearing %d from nohz_full range "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) "for timekeeping\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) cpumask_clear_cpu(cpu, tick_nohz_full_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) for_each_cpu(cpu, tick_nohz_full_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) context_tracking_cpu_set(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) "kernel/nohz:predown", NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) tick_nohz_cpu_down);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) WARN_ON(ret < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) cpumask_pr_args(tick_nohz_full_mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * NOHZ - aka dynamic tick functionality
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) #ifdef CONFIG_NO_HZ_COMMON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * NO HZ enabled ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) bool tick_nohz_enabled __read_mostly = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) unsigned long tick_nohz_active __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * Enable / Disable tickless mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) static int __init setup_tick_nohz(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) return (kstrtobool(str, &tick_nohz_enabled) == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) __setup("nohz=", setup_tick_nohz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) bool tick_nohz_tick_stopped(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return ts->tick_stopped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) bool tick_nohz_tick_stopped_cpu(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return ts->tick_stopped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * tick_nohz_update_jiffies - update jiffies when idle was interrupted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) * Called from interrupt entry when the CPU was idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * In case the sched_tick was stopped on this CPU, we have to check if jiffies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * must be updated. Otherwise an interrupt handler could use a stale jiffy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * value. We do this unconditionally on any CPU, as we don't know whether the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) * CPU, which has the update task assigned is in a long sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) static void tick_nohz_update_jiffies(ktime_t now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) __this_cpu_write(tick_cpu_sched.idle_waketime, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) tick_do_update_jiffies64(now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) touch_softlockup_watchdog_sched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) * Updates the per-CPU time idle statistics counters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) ktime_t delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) if (ts->idle_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) delta = ktime_sub(now, ts->idle_entrytime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) if (nr_iowait_cpu(cpu) > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) ts->idle_entrytime = now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (last_update_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) *last_update_time = ktime_to_us(now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) update_ts_time_stats(smp_processor_id(), ts, now, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) ts->idle_active = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) sched_clock_idle_wakeup_event();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) static void tick_nohz_start_idle(struct tick_sched *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) ts->idle_entrytime = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) ts->idle_active = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) sched_clock_idle_sleep_event();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * get_cpu_idle_time_us - get the total idle time of a CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * @cpu: CPU number to query
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * @last_update_time: variable to store update time in. Do not update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) * counters if NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) * Return the cumulative idle time (since boot) for a given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * CPU, in microseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) * This time is measured via accounting rather than sampling,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) * and is as accurate as ktime_get() is.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * This function returns -1 if NOHZ is not enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) ktime_t now, idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) if (!tick_nohz_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (last_update_time) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) update_ts_time_stats(cpu, ts, now, last_update_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) idle = ts->idle_sleeptime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) if (ts->idle_active && !nr_iowait_cpu(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) ktime_t delta = ktime_sub(now, ts->idle_entrytime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) idle = ktime_add(ts->idle_sleeptime, delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) idle = ts->idle_sleeptime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) return ktime_to_us(idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * get_cpu_iowait_time_us - get the total iowait time of a CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * @cpu: CPU number to query
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * @last_update_time: variable to store update time in. Do not update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * counters if NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * Return the cumulative iowait time (since boot) for a given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * CPU, in microseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * This time is measured via accounting rather than sampling,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * and is as accurate as ktime_get() is.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * This function returns -1 if NOHZ is not enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) ktime_t now, iowait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) if (!tick_nohz_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) if (last_update_time) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) update_ts_time_stats(cpu, ts, now, last_update_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) iowait = ts->iowait_sleeptime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) ktime_t delta = ktime_sub(now, ts->idle_entrytime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) iowait = ktime_add(ts->iowait_sleeptime, delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) iowait = ts->iowait_sleeptime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) return ktime_to_us(iowait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) hrtimer_cancel(&ts->sched_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) /* Forward the time to expire in the future */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) hrtimer_forward(&ts->sched_timer, now, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) hrtimer_start_expires(&ts->sched_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) HRTIMER_MODE_ABS_PINNED_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * Reset to make sure next tick stop doesn't get fooled by past
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * cached clock deadline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) ts->next_tick = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) static inline bool local_timer_softirq_pending(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return local_softirq_pending() & BIT(TIMER_SOFTIRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) u64 basemono, next_tick, next_tmr, next_rcu, delta, expires;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) unsigned long basejiff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) unsigned int seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) /* Read jiffies and the time when jiffies were updated last */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) seq = read_seqcount_begin(&jiffies_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) basemono = last_jiffies_update;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) basejiff = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) } while (read_seqcount_retry(&jiffies_seq, seq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) ts->last_jiffies = basejiff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) ts->timer_expires_base = basemono;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * Keep the periodic tick, when RCU, architecture or irq_work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * requests it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * Aside of that check whether the local timer softirq is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) * pending. If so its a bad idea to call get_next_timer_interrupt()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * because there is an already expired timer, so it will request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) * immeditate expiry, which rearms the hardware timer with a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) * minimal delta which brings us back to this place
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) * immediately. Lather, rinse and repeat...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) irq_work_needs_cpu() || local_timer_softirq_pending()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) next_tick = basemono + TICK_NSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) * Get the next pending timer. If high resolution
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * timers are enabled this only takes the timer wheel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * timers into account. If high resolution timers are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) * disabled this also looks at the next expiring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * hrtimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) next_tmr = get_next_timer_interrupt(basejiff, basemono);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) ts->next_timer = next_tmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) /* Take the next rcu event into account */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) next_tick = next_rcu < next_tmr ? next_rcu : next_tmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * If the tick is due in the next period, keep it ticking or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) * force prod the timer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) delta = next_tick - basemono;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (delta <= (u64)TICK_NSEC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) * Tell the timer code that the base is not idle, i.e. undo
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) * the effect of get_next_timer_interrupt():
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) timer_clear_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * We've not stopped the tick yet, and there's a timer in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * next period, so no point in stopping it either, bail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) if (!ts->tick_stopped) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) ts->timer_expires = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * If this CPU is the one which had the do_timer() duty last, we limit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * the sleep time to the timekeeping max_deferment value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * Otherwise we can sleep as long as we want.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) delta = timekeeping_max_deferment();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) if (cpu != tick_do_timer_cpu &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) delta = KTIME_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) /* Calculate the next expiry time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (delta < (KTIME_MAX - basemono))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) expires = basemono + delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) expires = KTIME_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) ts->timer_expires = min_t(u64, expires, next_tick);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return ts->timer_expires;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) u64 basemono = ts->timer_expires_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) u64 expires = ts->timer_expires;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) ktime_t tick = expires;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) /* Make sure we won't be trying to stop it twice in a row. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) ts->timer_expires_base = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * If this CPU is the one which updates jiffies, then give up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) * the assignment and let it be taken by the CPU which runs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) * the tick timer next, which might be this CPU as well. If we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) * don't drop this here the jiffies might be stale and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * do_timer() never invoked. Keep track of the fact that it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * was the one which had the do_timer() duty last.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (cpu == tick_do_timer_cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) tick_do_timer_cpu = TICK_DO_TIMER_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) ts->do_timer_last = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) ts->do_timer_last = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) /* Skip reprogram of event if its not changed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) if (ts->tick_stopped && (expires == ts->next_tick)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) /* Sanity check: make sure clockevent is actually programmed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) basemono, ts->next_tick, dev->next_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * nohz_stop_sched_tick can be called several times before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) * the nohz_restart_sched_tick is called. This happens when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * interrupts arrive which do not cause a reschedule. In the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * first call we save the current tick time, so we can restart
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * the scheduler tick in nohz_restart_sched_tick.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) if (!ts->tick_stopped) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) calc_load_nohz_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) quiet_vmstat();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) ts->tick_stopped = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) trace_tick_stop(1, TICK_DEP_MASK_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) ts->next_tick = tick;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) * If the expiration time == KTIME_MAX, then we simply stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) * the tick timer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) if (unlikely(expires == KTIME_MAX)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) hrtimer_cancel(&ts->sched_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) hrtimer_start(&ts->sched_timer, tick,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) HRTIMER_MODE_ABS_PINNED_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) hrtimer_set_expires(&ts->sched_timer, tick);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) tick_program_event(tick, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) static void tick_nohz_retain_tick(struct tick_sched *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) ts->timer_expires_base = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) #ifdef CONFIG_NO_HZ_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) if (tick_nohz_next_event(ts, cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) tick_nohz_stop_tick(ts, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) tick_nohz_retain_tick(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) #endif /* CONFIG_NO_HZ_FULL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) /* Update jiffies first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) tick_do_update_jiffies64(now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * Clear the timer idle flag, so we avoid IPIs on remote queueing and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * the clock forward checks in the enqueue path:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) timer_clear_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) calc_load_nohz_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) touch_softlockup_watchdog_sched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * Cancel the scheduled timer and restore the tick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) ts->tick_stopped = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) ts->idle_exittime = now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) tick_nohz_restart(ts, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) static void tick_nohz_full_update_tick(struct tick_sched *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) #ifdef CONFIG_NO_HZ_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (!tick_nohz_full_cpu(cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) if (can_stop_full_tick(cpu, ts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) tick_nohz_stop_sched_tick(ts, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) else if (ts->tick_stopped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) tick_nohz_restart_sched_tick(ts, ktime_get());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * If this CPU is offline and it is the one which updates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) * jiffies, then give up the assignment and let it be taken by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) * the CPU which runs the tick timer next. If we don't drop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) * this here the jiffies might be stale and do_timer() never
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) * invoked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) if (unlikely(!cpu_online(cpu))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) if (cpu == tick_do_timer_cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) tick_do_timer_cpu = TICK_DO_TIMER_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * Make sure the CPU doesn't get fooled by obsolete tick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * deadline if it comes back online later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) ts->next_tick = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) if (need_resched())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) if (unlikely(local_softirq_pending())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) static int ratelimit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) if (ratelimit < 10 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) pr_warn("NOHZ tick-stop error: Non-RCU local softirq work is pending, handler #%02x!!!\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) (unsigned int) local_softirq_pending());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) ratelimit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) if (tick_nohz_full_enabled()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) * Keep the tick alive to guarantee timekeeping progression
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * if there are full dynticks CPUs around
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) if (tick_do_timer_cpu == cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) /* Should not happen for nohz-full */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) if (WARN_ON_ONCE(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) ktime_t expires;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) * tick timer expiration time is known already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if (ts->timer_expires_base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) expires = ts->timer_expires;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) else if (can_stop_idle_tick(cpu, ts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) expires = tick_nohz_next_event(ts, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) ts->idle_calls++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) if (expires > 0LL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) int was_stopped = ts->tick_stopped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) tick_nohz_stop_tick(ts, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) ts->idle_sleeps++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) ts->idle_expires = expires;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) if (!was_stopped && ts->tick_stopped) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) ts->idle_jiffies = ts->last_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) nohz_balance_enter_idle(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) tick_nohz_retain_tick(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) * When the next event is more than a tick into the future, stop the idle tick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) void tick_nohz_idle_stop_tick(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) void tick_nohz_idle_retain_tick(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) * Undo the effect of get_next_timer_interrupt() called from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) * tick_nohz_next_event().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) timer_clear_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) * tick_nohz_idle_enter - prepare for entering idle on the current CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * Called when we start the idle loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) void tick_nohz_idle_enter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) struct tick_sched *ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) lockdep_assert_irqs_enabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) WARN_ON_ONCE(ts->timer_expires_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) ts->inidle = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) tick_nohz_start_idle(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) * tick_nohz_irq_exit - update next tick event from interrupt exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) * When an interrupt fires while we are idle and it doesn't cause
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) * a reschedule, it may still add, modify or delete a timer, enqueue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) * an RCU callback, etc...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * So we need to re-calculate and reprogram the next tick event.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) void tick_nohz_irq_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) if (ts->inidle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) tick_nohz_start_idle(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) tick_nohz_full_update_tick(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) * tick_nohz_idle_got_tick - Check whether or not the tick handler has run
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) bool tick_nohz_idle_got_tick(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) if (ts->got_idle_tick) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) ts->got_idle_tick = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) * or the tick, whatever that expires first. Note that, if the tick has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) * stopped, it returns the next hrtimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) * Called from power state control code with interrupts disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) ktime_t tick_nohz_get_next_hrtimer(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) return __this_cpu_read(tick_cpu_device.evtdev)->next_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) * tick_nohz_get_sleep_length - return the expected length of the current sleep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) * @delta_next: duration until the next event if the tick cannot be stopped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) * Called from power state control code with interrupts disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) * The idle entry time is expected to be a sufficient approximation of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * the current time at this point.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) ktime_t now = ts->idle_entrytime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) ktime_t next_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) WARN_ON_ONCE(!ts->inidle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) *delta_next = ktime_sub(dev->next_event, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) if (!can_stop_idle_tick(cpu, ts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) return *delta_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) next_event = tick_nohz_next_event(ts, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) if (!next_event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) return *delta_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) * If the next highres timer to expire is earlier than next_event, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) * idle governor needs to know that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) next_event = min_t(u64, next_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) hrtimer_next_event_without(&ts->sched_timer));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) return ktime_sub(next_event, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) EXPORT_SYMBOL_GPL(tick_nohz_get_sleep_length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) * for a particular CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) * Called from the schedutil frequency scaling governor in scheduler context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) struct tick_sched *ts = tick_get_tick_sched(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) return ts->idle_calls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) EXPORT_SYMBOL_GPL(tick_nohz_get_idle_calls_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) * tick_nohz_get_idle_calls - return the current idle calls counter value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * Called from the schedutil frequency scaling governor in scheduler context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) unsigned long tick_nohz_get_idle_calls(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) return ts->idle_calls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) unsigned long ticks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) if (vtime_accounting_enabled_this_cpu())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) * We stopped the tick in idle. Update process times would miss the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) * time we slept as update_process_times does only a 1 tick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) * accounting. Enforce that this is accounted to idle !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) ticks = jiffies - ts->idle_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) * We might be one off. Do not randomly account a huge number of ticks!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) if (ticks && ticks < LONG_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) account_idle_ticks(ticks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) tick_nohz_restart_sched_tick(ts, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) tick_nohz_account_idle_ticks(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) void tick_nohz_idle_restart_tick(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) if (ts->tick_stopped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) __tick_nohz_idle_restart_tick(ts, ktime_get());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) * tick_nohz_idle_exit - restart the idle tick from the idle task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) * Restart the idle tick when the CPU is woken up from idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) * This also exit the RCU extended quiescent state. The CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) * can use RCU again after this function is called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) void tick_nohz_idle_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) bool idle_active, tick_stopped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) ktime_t now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) WARN_ON_ONCE(!ts->inidle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) WARN_ON_ONCE(ts->timer_expires_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) ts->inidle = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) idle_active = ts->idle_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) tick_stopped = ts->tick_stopped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) if (idle_active || tick_stopped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) if (idle_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) tick_nohz_stop_idle(ts, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) if (tick_stopped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) __tick_nohz_idle_restart_tick(ts, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) * The nohz low res interrupt handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) static void tick_nohz_handler(struct clock_event_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) struct pt_regs *regs = get_irq_regs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) ktime_t now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) dev->next_event = KTIME_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) tick_sched_do_timer(ts, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) tick_sched_handle(ts, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) /* No need to reprogram if we are running tickless */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) if (unlikely(ts->tick_stopped))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) hrtimer_forward(&ts->sched_timer, now, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) if (!tick_nohz_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) ts->nohz_mode = mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) /* One update is enough */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) if (!test_and_set_bit(0, &tick_nohz_active))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) timers_update_nohz();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) * tick_nohz_switch_to_nohz - switch to nohz mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) static void tick_nohz_switch_to_nohz(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) ktime_t next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) if (!tick_nohz_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) if (tick_switch_to_oneshot(tick_nohz_handler))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) * Recycle the hrtimer in ts, so we can share the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) * hrtimer_forward with the highres code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) /* Get the next period */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) next = tick_init_jiffy_update();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) hrtimer_set_expires(&ts->sched_timer, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) hrtimer_forward_now(&ts->sched_timer, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) static inline void tick_nohz_irq_enter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) ktime_t now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) if (!ts->idle_active && !ts->tick_stopped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) if (ts->idle_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) tick_nohz_stop_idle(ts, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) if (ts->tick_stopped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) tick_nohz_update_jiffies(now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) static inline void tick_nohz_switch_to_nohz(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) static inline void tick_nohz_irq_enter(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) #endif /* CONFIG_NO_HZ_COMMON */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * Called from irq_enter to notify about the possible interruption of idle()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) void tick_irq_enter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) tick_check_oneshot_broadcast_this_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) tick_nohz_irq_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) * High resolution timer specific code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) #ifdef CONFIG_HIGH_RES_TIMERS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) * We rearm the timer until we get disabled by the idle code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) * Called with interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) struct tick_sched *ts =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) container_of(timer, struct tick_sched, sched_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) struct pt_regs *regs = get_irq_regs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) ktime_t now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) tick_sched_do_timer(ts, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) * Do not call, when we are not in irq context and have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) * no valid regs pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) if (regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) tick_sched_handle(ts, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) ts->next_tick = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) /* No need to reprogram if we are in idle or full dynticks mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) if (unlikely(ts->tick_stopped))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) return HRTIMER_NORESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) hrtimer_forward(timer, now, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) return HRTIMER_RESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) static int sched_skew_tick;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) static int __init skew_tick(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) get_option(&str, &sched_skew_tick);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) early_param("skew_tick", skew_tick);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) * tick_setup_sched_timer - setup the tick emulation timer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) void tick_setup_sched_timer(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) ktime_t now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) * Emulate tick processing via per-CPU hrtimers:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) ts->sched_timer.function = tick_sched_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) /* Get the next period (per-CPU) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) /* Offset the tick to avert jiffies_lock contention. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) if (sched_skew_tick) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) u64 offset = ktime_to_ns(tick_period) >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) do_div(offset, num_possible_cpus());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) offset *= smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) hrtimer_add_expires_ns(&ts->sched_timer, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) hrtimer_forward(&ts->sched_timer, now, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) #endif /* HIGH_RES_TIMERS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) #if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) void tick_cancel_sched_timer(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) # ifdef CONFIG_HIGH_RES_TIMERS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) if (ts->sched_timer.base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) hrtimer_cancel(&ts->sched_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) # endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) memset(ts, 0, sizeof(*ts));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) * Async notification about clocksource changes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) void tick_clock_notify(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) * Async notification about clock event changes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) void tick_oneshot_notify(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) set_bit(0, &ts->check_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) * Check, if a change happened, which makes oneshot possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) * Called cyclic from the hrtimer softirq (driven by the timer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) * softirq) allow_nohz signals, that we can switch into low-res nohz
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) * mode, because high resolution timers are disabled (either compile
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) * or runtime). Called with interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) int tick_check_oneshot_change(int allow_nohz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) if (!test_and_clear_bit(0, &ts->check_clocks))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) if (!allow_nohz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) tick_nohz_switch_to_nohz();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags