Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Generic entry points for the idle threads and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * implementation of the idle task scheduling class.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * (NOTE: these are not related to SCHED_IDLE batch scheduled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *        tasks which are handled in sched/fair.c )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include "sched.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <trace/events/power.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <trace/hooks/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) /* Linker adds these: start and end of __cpuidle functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) extern char __cpuidle_text_start[], __cpuidle_text_end[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  * sched_idle_set_state - Record idle state for the current CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  * @idle_state: State to record.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) void sched_idle_set_state(struct cpuidle_state *idle_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	idle_set_state(this_rq(), idle_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) static int __read_mostly cpu_idle_force_poll;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) void cpu_idle_poll_ctrl(bool enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	if (enable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 		cpu_idle_force_poll++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 		cpu_idle_force_poll--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 		WARN_ON_ONCE(cpu_idle_force_poll < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) #ifdef CONFIG_GENERIC_IDLE_POLL_SETUP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) static int __init cpu_idle_poll_setup(char *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	cpu_idle_force_poll = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) __setup("nohlt", cpu_idle_poll_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) static int __init cpu_idle_nopoll_setup(char *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	cpu_idle_force_poll = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) __setup("hlt", cpu_idle_nopoll_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) static noinline int __cpuidle cpu_idle_poll(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	trace_cpu_idle(0, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	stop_critical_timings();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	rcu_idle_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	while (!tif_need_resched() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	       (cpu_idle_force_poll || tick_check_broadcast_expired()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 		cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	rcu_idle_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	start_critical_timings();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) /* Weak implementations for optional arch specific functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) void __weak arch_cpu_idle_prepare(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) void __weak arch_cpu_idle_enter(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) void __weak arch_cpu_idle_exit(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) void __weak arch_cpu_idle_dead(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) void __weak arch_cpu_idle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	cpu_idle_force_poll = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	raw_local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87)  * default_idle_call - Default CPU idle routine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89)  * To use when the cpuidle framework cannot be used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) void __cpuidle default_idle_call(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	if (current_clr_polling_and_test()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 		local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 		trace_cpu_idle(1, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 		stop_critical_timings();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 		 * arch_cpu_idle() is supposed to enable IRQs, however
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 		 * we can't do that because of RCU and tracing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 		 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 		 * Trace IRQs enable here, then switch off RCU, and have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		 * arch_cpu_idle() use raw_local_irq_enable(). Note that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		 * rcu_idle_enter() relies on lockdep IRQ state, so switch that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 		 * last -- this is very similar to the entry code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 		trace_hardirqs_on_prepare();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 		lockdep_hardirqs_on_prepare(_THIS_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 		rcu_idle_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 		lockdep_hardirqs_on(_THIS_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 		arch_cpu_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 		 * OK, so IRQs are enabled here, but RCU needs them disabled to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 		 * turn itself back on.. funny thing is that disabling IRQs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 		 * will cause tracing, which needs RCU. Jump through hoops to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 		 * make it 'work'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 		raw_local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 		lockdep_hardirqs_off(_THIS_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 		rcu_idle_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 		lockdep_hardirqs_on(_THIS_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 		raw_local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		start_critical_timings();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static int call_cpuidle_s2idle(struct cpuidle_driver *drv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 			       struct cpuidle_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	if (current_clr_polling_and_test())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 		return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	return cpuidle_enter_s2idle(drv, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 		      int next_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	 * The idle task must be scheduled, it is pointless to go to idle, just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	 * update no idle residency and return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	if (current_clr_polling_and_test()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 		dev->last_residency_ns = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 		local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 		return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	 * Enter the idle state previously returned by the governor decision.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	 * This function will block until an interrupt occurs and will take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	 * care of re-enabling the local interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	return cpuidle_enter(drv, dev, next_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)  * cpuidle_idle_call - the main idle function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)  * NOTE: no locks or semaphores should be used here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)  * On archs that support TIF_POLLING_NRFLAG, is called with polling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)  * set, and it returns with polling set.  If it ever stops polling, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)  * must clear the polling bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static void cpuidle_idle_call(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	struct cpuidle_device *dev = cpuidle_get_device();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	int next_state, entered_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	 * Check if the idle task must be rescheduled. If it is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	 * case, exit the function after re-enabling the local irq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	if (need_resched()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 		local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	 * The RCU framework needs to be told that we are entering an idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	 * section, so no more rcu read side critical sections and one more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	 * step to the grace period
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	if (cpuidle_not_available(drv, dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 		tick_nohz_idle_stop_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 		default_idle_call();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 		goto exit_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	 * Suspend-to-idle ("s2idle") is a system state in which all user space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	 * has been frozen, all I/O devices have been suspended and the only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	 * activity happens here and in interrupts (if any). In that case bypass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	 * the cpuidle governor and go stratight for the deepest idle state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	 * available.  Possibly also suspend the local tick and the entire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	 * timekeeping to prevent timer interrupts from kicking us out of idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	 * until a proper wakeup interrupt happens.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	if (idle_should_enter_s2idle() || dev->forced_idle_latency_limit_ns) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 		u64 max_latency_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 		if (idle_should_enter_s2idle()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 			entered_state = call_cpuidle_s2idle(drv, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 			if (entered_state > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 				goto exit_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 			max_latency_ns = U64_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 		} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 			max_latency_ns = dev->forced_idle_latency_limit_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 		tick_nohz_idle_stop_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 		next_state = cpuidle_find_deepest_state(drv, dev, max_latency_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 		call_cpuidle(drv, dev, next_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 		bool stop_tick = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 		 * Ask the cpuidle framework to choose a convenient idle state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 		next_state = cpuidle_select(drv, dev, &stop_tick);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 		if (stop_tick || tick_nohz_tick_stopped())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 			tick_nohz_idle_stop_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 			tick_nohz_idle_retain_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		entered_state = call_cpuidle(drv, dev, next_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		 * Give the governor an opportunity to reflect on the outcome
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 		cpuidle_reflect(dev, entered_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) exit_idle:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	__current_set_polling();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	 * It is up to the idle functions to reenable local interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	if (WARN_ON_ONCE(irqs_disabled()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 		local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)  * Generic idle loop implementation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)  * Called with polling cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static void do_idle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	 * If the arch has a polling bit, we maintain an invariant:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	 * Our polling bit is clear if we're not scheduled (i.e. if rq->curr !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	 * rq->idle). This means that, if rq->idle has the polling bit set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	 * then setting need_resched is guaranteed to cause the CPU to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	 * reschedule.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	__current_set_polling();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	tick_nohz_idle_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	while (!need_resched()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 		rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 		local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 		if (cpu_is_offline(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 			tick_nohz_idle_stop_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 			cpuhp_report_idle_dead();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 			arch_cpu_idle_dead();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 		arch_cpu_idle_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 		rcu_nocb_flush_deferred_wakeup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 		 * In poll mode we reenable interrupts and spin. Also if we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 		 * detected in the wakeup from idle path that the tick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 		 * broadcast device expired for us, we don't want to go deep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 		 * idle as we know that the IPI is going to arrive right away.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 		if (cpu_idle_force_poll || tick_check_broadcast_expired()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 			tick_nohz_idle_restart_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 			cpu_idle_poll();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 		} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 			cpuidle_idle_call();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 		arch_cpu_idle_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	 * Since we fell out of the loop above, we know TIF_NEED_RESCHED must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	 * be set, propagate it into PREEMPT_NEED_RESCHED.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	 * This is required because for polling idle loops we will not have had
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	 * an IPI to fold the state for us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	preempt_set_need_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	tick_nohz_idle_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	__current_clr_polling();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	 * We promise to call sched_ttwu_pending() and reschedule if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	 * need_resched() is set while polling is set. That means that clearing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	 * polling needs to be visible before doing these things.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	 * RCU relies on this call to be done outside of an RCU read-side
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	 * critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	flush_smp_call_function_from_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	schedule_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	if (unlikely(klp_patch_pending(current)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 		klp_update_patch_state(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) bool cpu_in_idle(unsigned long pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	return pc >= (unsigned long)__cpuidle_text_start &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 		pc < (unsigned long)__cpuidle_text_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) struct idle_timer {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	struct hrtimer timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	int done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	struct idle_timer *it = container_of(timer, struct idle_timer, timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	WRITE_ONCE(it->done, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	set_tsk_need_resched(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	return HRTIMER_NORESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) void play_idle_precise(u64 duration_ns, u64 latency_ns)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	struct idle_timer it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	 * Only FIFO tasks can disable the tick since they don't need the forced
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 	 * preemption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 	WARN_ON_ONCE(current->policy != SCHED_FIFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	WARN_ON_ONCE(current->nr_cpus_allowed != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	WARN_ON_ONCE(!(current->flags & PF_KTHREAD));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 	WARN_ON_ONCE(!(current->flags & PF_NO_SETAFFINITY));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	WARN_ON_ONCE(!duration_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	rcu_sleep_check();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	current->flags |= PF_IDLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	cpuidle_use_deepest_state(latency_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	it.done = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 	hrtimer_init_on_stack(&it.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	it.timer.function = idle_inject_timer_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 	hrtimer_start(&it.timer, ns_to_ktime(duration_ns),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 		      HRTIMER_MODE_REL_PINNED_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 	while (!READ_ONCE(it.done))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 		do_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 	cpuidle_use_deepest_state(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	current->flags &= ~PF_IDLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 	preempt_fold_need_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 	preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) EXPORT_SYMBOL_GPL(play_idle_precise);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) void cpu_startup_entry(enum cpuhp_state state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 	arch_cpu_idle_prepare();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 	cpuhp_online_idle(state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 	while (1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 		do_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)  * idle-task scheduling class.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) select_task_rq_idle(struct task_struct *p, int cpu, int sd_flag, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	return task_cpu(p); /* IDLE tasks as never migrated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) balance_idle(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 	return WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)  * Idle tasks are unconditionally rescheduled:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 	resched_curr(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) static void put_prev_task_idle(struct rq *rq, struct task_struct *prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) 	update_idle_core(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) 	schedstat_inc(rq->sched_goidle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct task_struct *pick_next_task_idle(struct rq *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 	struct task_struct *next = rq->idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 	set_next_task_idle(rq, next, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 	return next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)  * It is not legal to sleep in the idle task - print a warning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)  * message if some code attempts to do it:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) dequeue_task_idle(struct rq *rq, struct task_struct *p, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) 	raw_spin_unlock_irq(&rq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) 	printk(KERN_ERR "bad: scheduling from the idle thread!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) 	trace_android_rvh_dequeue_task_idle(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) 	dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) 	raw_spin_lock_irq(&rq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)  * scheduler tick hitting a task of our scheduling class.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)  * NOTE: This function can be called remotely by the tick offload that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)  * goes along full dynticks. Therefore no local assumption can be made
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)  * and everything must be accessed through the @rq and @curr passed in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)  * parameters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) static void task_tick_idle(struct rq *rq, struct task_struct *curr, int queued)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) static void switched_to_idle(struct rq *rq, struct task_struct *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) 	BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) prio_changed_idle(struct rq *rq, struct task_struct *p, int oldprio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 	BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) static void update_curr_idle(struct rq *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)  * Simple, special scheduling class for the per-CPU idle tasks:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) const struct sched_class idle_sched_class
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 	__section("__idle_sched_class") = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) 	/* no enqueue/yield_task for idle tasks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) 	/* dequeue is not valid, we print a debug message there: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) 	.dequeue_task		= dequeue_task_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 	.check_preempt_curr	= check_preempt_curr_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) 	.pick_next_task		= pick_next_task_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) 	.put_prev_task		= put_prev_task_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) 	.set_next_task          = set_next_task_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 	.balance		= balance_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) 	.select_task_rq		= select_task_rq_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) 	.set_cpus_allowed	= set_cpus_allowed_common,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) 	.task_tick		= task_tick_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) 	.prio_changed		= prio_changed_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) 	.switched_to		= switched_to_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) 	.update_curr		= update_curr_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) };