Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * This file contains the base functions to manage periodic tick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * related events.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/hrtimer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <linux/profile.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <trace/events/power.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <trace/hooks/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <asm/irq_regs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include "tick-internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  * Tick devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) DEFINE_PER_CPU(struct tick_device, tick_cpu_device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  * Tick next event: keeps track of the tick time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) ktime_t tick_next_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) ktime_t tick_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  * tick_do_timer_cpu is a timer core internal variable which holds the CPU NR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38)  * which is responsible for calling do_timer(), i.e. the timekeeping stuff. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  * variable has two functions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  * 1) Prevent a thundering herd issue of a gazillion of CPUs trying to grab the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42)  *    timekeeping lock all at once. Only the CPU which is assigned to do the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43)  *    update is handling it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  * 2) Hand off the duty in the NOHZ idle case by setting the value to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  *    TICK_DO_TIMER_NONE, i.e. a non existing CPU. So the next cpu which looks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  *    at it will take over and keep the time keeping alive.  The handover
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48)  *    procedure also covers cpu hotplug.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) #ifdef CONFIG_NO_HZ_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53)  * tick_do_timer_boot_cpu indicates the boot CPU temporarily owns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54)  * tick_do_timer_cpu and it should be taken over by an eligible secondary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55)  * when one comes online.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) static int tick_do_timer_boot_cpu __read_mostly = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61)  * Debugging: see timer_list.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) struct tick_device *tick_get_device(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	return &per_cpu(tick_cpu_device, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69)  * tick_is_oneshot_available - check for a oneshot capable event device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) int tick_is_oneshot_available(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	if (!dev || !(dev->features & CLOCK_EVT_FEAT_ONESHOT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 		return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	return tick_broadcast_oneshot_available();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83)  * Periodic tick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) static void tick_periodic(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	if (tick_do_timer_cpu == cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 		raw_spin_lock(&jiffies_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		write_seqcount_begin(&jiffies_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 		/* Keep track of the next tick event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 		tick_next_period = ktime_add(tick_next_period, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 		do_timer(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 		write_seqcount_end(&jiffies_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 		raw_spin_unlock(&jiffies_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 		update_wall_time();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 		trace_android_vh_jiffies_update(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	update_process_times(user_mode(get_irq_regs()));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	profile_tick(CPU_PROFILING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)  * Event handler for periodic ticks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) void tick_handle_periodic(struct clock_event_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	ktime_t next = dev->next_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	tick_periodic(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #if defined(CONFIG_HIGH_RES_TIMERS) || defined(CONFIG_NO_HZ_COMMON)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	 * The cpu might have transitioned to HIGHRES or NOHZ mode via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	 * update_process_times() -> run_local_timers() ->
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	 * hrtimer_run_queues().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	if (dev->event_handler != tick_handle_periodic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	if (!clockevent_state_oneshot(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		 * Setup the next period for devices, which do not have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 		 * periodic mode:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 		next = ktime_add(next, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 		if (!clockevents_program_event(dev, next, false))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 			return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 		 * Have to be careful here. If we're in oneshot mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 		 * before we call tick_periodic() in a loop, we need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 		 * to be sure we're using a real hardware clocksource.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 		 * Otherwise we could get trapped in an infinite
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 		 * loop, as the tick_periodic() increments jiffies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 		 * which then will increment time, possibly causing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 		 * the loop to trigger again and again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 		if (timekeeping_valid_for_hres())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 			tick_periodic(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)  * Setup the device for a periodic tick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	tick_set_periodic_handler(dev, broadcast);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	/* Broadcast setup ? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	if (!tick_device_is_functional(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	if ((dev->features & CLOCK_EVT_FEAT_PERIODIC) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	    !tick_broadcast_oneshot_active()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 		clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 		unsigned int seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		ktime_t next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 		do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 			seq = read_seqcount_begin(&jiffies_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 			next = tick_next_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 		} while (read_seqcount_retry(&jiffies_seq, seq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 		clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 		for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 			if (!clockevents_program_event(dev, next, false))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 				return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 			next = ktime_add(next, tick_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #ifdef CONFIG_NO_HZ_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static void giveup_do_timer(void *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	int cpu = *(unsigned int *)info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	WARN_ON(tick_do_timer_cpu != smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	tick_do_timer_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) static void tick_take_do_timer_from_boot(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	int from = tick_do_timer_boot_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	if (from >= 0 && from != cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 		smp_call_function_single(from, giveup_do_timer, &cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)  * Setup the tick device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) static void tick_setup_device(struct tick_device *td,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 			      struct clock_event_device *newdev, int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 			      const struct cpumask *cpumask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	void (*handler)(struct clock_event_device *) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	ktime_t next_event = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	 * First device setup ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	if (!td->evtdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 		 * If no cpu took the do_timer update, assign it to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 		 * this cpu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 		if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 			tick_do_timer_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 			tick_next_period = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 			tick_period = NSEC_PER_SEC / HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #ifdef CONFIG_NO_HZ_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 			/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 			 * The boot CPU may be nohz_full, in which case set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 			 * tick_do_timer_boot_cpu so the first housekeeping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 			 * secondary that comes up will take do_timer from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 			 * us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 			 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 			if (tick_nohz_full_cpu(cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 				tick_do_timer_boot_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 		} else if (tick_do_timer_boot_cpu != -1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 						!tick_nohz_full_cpu(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 			tick_take_do_timer_from_boot();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 			tick_do_timer_boot_cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 			WARN_ON(tick_do_timer_cpu != cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 		 * Startup in periodic mode first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 		td->mode = TICKDEV_MODE_PERIODIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 		handler = td->evtdev->event_handler;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 		next_event = td->evtdev->next_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 		td->evtdev->event_handler = clockevents_handle_noop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	td->evtdev = newdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	 * When the device is not per cpu, pin the interrupt to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	 * current cpu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	if (!cpumask_equal(newdev->cpumask, cpumask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 		irq_set_affinity(newdev->irq, cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	 * When global broadcasting is active, check if the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	 * device is registered as a placeholder for broadcast mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	 * This allows us to handle this x86 misfeature in a generic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	 * way. This function also returns !=0 when we keep the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	 * current active broadcast state for this CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	if (tick_device_uses_broadcast(newdev, cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	if (td->mode == TICKDEV_MODE_PERIODIC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 		tick_setup_periodic(newdev, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 		tick_setup_oneshot(newdev, handler, next_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) void tick_install_replacement(struct clock_event_device *newdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	clockevents_exchange_device(td->evtdev, newdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 		tick_oneshot_notify();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static bool tick_check_percpu(struct clock_event_device *curdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 			      struct clock_event_device *newdev, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	if (!cpumask_test_cpu(cpu, newdev->cpumask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	if (cpumask_equal(newdev->cpumask, cpumask_of(cpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 		return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	/* Check if irq affinity can be set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	if (newdev->irq >= 0 && !irq_can_set_affinity(newdev->irq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	/* Prefer an existing cpu local device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) static bool tick_check_preferred(struct clock_event_device *curdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 				 struct clock_event_device *newdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	/* Prefer oneshot capable device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	if (!(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 		if (curdev && (curdev->features & CLOCK_EVT_FEAT_ONESHOT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 			return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 		if (tick_oneshot_mode_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 			return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	 * Use the higher rated one, but prefer a CPU local device with a lower
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	 * rating than a non-CPU local device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	return !curdev ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 		newdev->rating > curdev->rating ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	       !cpumask_equal(curdev->cpumask, newdev->cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)  * Check whether the new device is a better fit than curdev. curdev
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)  * can be NULL !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) bool tick_check_replacement(struct clock_event_device *curdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 			    struct clock_event_device *newdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 	if (!tick_check_percpu(curdev, newdev, smp_processor_id()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	return tick_check_preferred(curdev, newdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)  * Check, if the new registered device should be used. Called with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)  * clockevents_lock held and interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) void tick_check_new_device(struct clock_event_device *newdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	struct clock_event_device *curdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 	struct tick_device *td;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	td = &per_cpu(tick_cpu_device, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	curdev = td->evtdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	/* cpu local device ? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	if (!tick_check_percpu(curdev, newdev, cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 		goto out_bc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 	/* Preference decision */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 	if (!tick_check_preferred(curdev, newdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 		goto out_bc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	if (!try_module_get(newdev->owner))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 	 * Replace the eventually existing device by the new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	 * device. If the current device is the broadcast device, do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	 * not give it back to the clockevents layer !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	if (tick_is_broadcast_device(curdev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 		clockevents_shutdown(curdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 		curdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	clockevents_exchange_device(curdev, newdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 	if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 		tick_oneshot_notify();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 	return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) out_bc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 	 * Can the new device be used as a broadcast device ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 	tick_install_broadcast_device(newdev, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)  * tick_broadcast_oneshot_control - Enter/exit broadcast oneshot mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)  * @state:	The target state (enter/exit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)  * The system enters/leaves a state, where affected devices might stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)  * Returns 0 on success, -EBUSY if the cpu is used to broadcast wakeups.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)  * Called with interrupts disabled, so clockevents_lock is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)  * required here because the local clock event device cannot go away
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)  * under us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) int tick_broadcast_oneshot_control(enum tick_broadcast_state state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) 	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 	if (!(td->evtdev->features & CLOCK_EVT_FEAT_C3STOP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) 	return __tick_broadcast_oneshot_control(state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) EXPORT_SYMBOL_GPL(tick_broadcast_oneshot_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)  * Transfer the do_timer job away from a dying cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)  * Called with interrupts disabled. Not locking required. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)  * tick_do_timer_cpu is owned by this cpu, nothing can change it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) void tick_handover_do_timer(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 	if (tick_do_timer_cpu == smp_processor_id()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 		int cpu = cpumask_first(cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 		tick_do_timer_cpu = (cpu < nr_cpu_ids) ? cpu :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 			TICK_DO_TIMER_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)  * Shutdown an event device on a given cpu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)  * This is called on a life CPU, when a CPU is dead. So we cannot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)  * access the hardware device itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)  * We just set the mode and remove it from the lists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) void tick_shutdown(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) 	struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 	struct clock_event_device *dev = td->evtdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 	td->mode = TICKDEV_MODE_PERIODIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 	if (dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 		 * Prevent that the clock events layer tries to call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 		 * the set mode function!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 		clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 		clockevents_exchange_device(dev, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 		dev->event_handler = clockevents_handle_noop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) 		td->evtdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)  * tick_suspend_local - Suspend the local tick device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)  * Called from the local cpu for freeze with interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)  * No locks required. Nothing can change the per cpu device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) void tick_suspend_local(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) 	clockevents_shutdown(td->evtdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)  * tick_resume_local - Resume the local tick device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)  * Called from the local CPU for unfreeze or XEN resume magic.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)  * No locks required. Nothing can change the per cpu device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) void tick_resume_local(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) 	bool broadcast = tick_resume_check_broadcast();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) 	clockevents_tick_resume(td->evtdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) 	if (!broadcast) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) 		if (td->mode == TICKDEV_MODE_PERIODIC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 			tick_setup_periodic(td->evtdev, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) 			tick_resume_oneshot();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)  * tick_suspend - Suspend the tick and the broadcast device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)  * Called from syscore_suspend() via timekeeping_suspend with only one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)  * CPU online and interrupts disabled or from tick_unfreeze() under
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)  * tick_freeze_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)  * No locks required. Nothing can change the per cpu device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) void tick_suspend(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 	tick_suspend_local();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 	tick_suspend_broadcast();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)  * tick_resume - Resume the tick and the broadcast device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)  * Called from syscore_resume() via timekeeping_resume with only one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)  * CPU online and interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)  * No locks required. Nothing can change the per cpu device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) void tick_resume(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) 	tick_resume_broadcast();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) 	tick_resume_local();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) #ifdef CONFIG_SUSPEND
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) static DEFINE_RAW_SPINLOCK(tick_freeze_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) static unsigned int tick_freeze_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)  * tick_freeze - Suspend the local tick and (possibly) timekeeping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)  * Check if this is the last online CPU executing the function and if so,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)  * suspend timekeeping.  Otherwise suspend the local tick.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)  * Call with interrupts disabled.  Must be balanced with %tick_unfreeze().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)  * Interrupts must not be enabled before the subsequent %tick_unfreeze().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) void tick_freeze(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) 	raw_spin_lock(&tick_freeze_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) 	tick_freeze_depth++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) 	if (tick_freeze_depth == num_online_cpus()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) 		trace_suspend_resume(TPS("timekeeping_freeze"),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) 				     smp_processor_id(), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) 		system_state = SYSTEM_SUSPEND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) 		sched_clock_suspend();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) 		timekeeping_suspend();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) 		tick_suspend_local();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) 	raw_spin_unlock(&tick_freeze_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)  * tick_unfreeze - Resume the local tick and (possibly) timekeeping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)  * Check if this is the first CPU executing the function and if so, resume
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)  * timekeeping.  Otherwise resume the local tick.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)  * Call with interrupts disabled.  Must be balanced with %tick_freeze().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)  * Interrupts must not be enabled after the preceding %tick_freeze().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) void tick_unfreeze(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) 	raw_spin_lock(&tick_freeze_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) 	if (tick_freeze_depth == num_online_cpus()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) 		timekeeping_resume();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) 		sched_clock_resume();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) 		system_state = SYSTEM_RUNNING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) 		trace_suspend_resume(TPS("timekeeping_freeze"),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) 				     smp_processor_id(), false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) 		touch_softlockup_watchdog();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) 		tick_resume_local();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) 	tick_freeze_depth--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) 	raw_spin_unlock(&tick_freeze_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) #endif /* CONFIG_SUSPEND */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)  * tick_init - initialize the tick control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) void __init tick_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) 	tick_broadcast_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) 	tick_nohz_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }