^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * ladder.c - the residency ladder algorithm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2004, 2005 Dominik Brodowski <linux@brodo.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Shaohua Li <shaohua.li@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Adam Belay <abelay@novell.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * This code is licenced under the GPL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/cpuidle.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/tick.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define PROMOTION_COUNT 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define DEMOTION_COUNT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct ladder_device_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) u32 promotion_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) u32 demotion_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) u64 promotion_time_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) u64 demotion_time_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) } threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) int promotion_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) int demotion_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) } stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct ladder_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct ladder_device_state states[CPUIDLE_STATE_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static DEFINE_PER_CPU(struct ladder_device, ladder_devices);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * ladder_do_selection - prepares private data for a state change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * @ldev: the ladder device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * @old_idx: the current state index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * @new_idx: the new target state index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static inline void ladder_do_selection(struct cpuidle_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct ladder_device *ldev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) int old_idx, int new_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) ldev->states[old_idx].stats.promotion_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) ldev->states[old_idx].stats.demotion_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) dev->last_state_idx = new_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * ladder_select_state - selects the next state to enter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * @drv: cpuidle driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * @dev: the CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * @dummy: not used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static int ladder_select_state(struct cpuidle_driver *drv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct cpuidle_device *dev, bool *dummy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct ladder_device_state *last_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) int last_idx = dev->last_state_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) s64 latency_req = cpuidle_governor_latency_req(dev->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) s64 last_residency;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /* Special case when user has set very strict latency requirement */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) if (unlikely(latency_req == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) ladder_do_selection(dev, ldev, last_idx, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) last_state = &ldev->states[last_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) last_residency = dev->last_residency_ns - drv->states[last_idx].exit_latency_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) /* consider promotion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (last_idx < drv->state_count - 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) !dev->states_usage[last_idx + 1].disable &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) last_residency > last_state->threshold.promotion_time_ns &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) drv->states[last_idx + 1].exit_latency_ns <= latency_req) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) last_state->stats.promotion_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) last_state->stats.demotion_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) ladder_do_selection(dev, ldev, last_idx, last_idx + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return last_idx + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /* consider demotion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (last_idx > first_idx &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) (dev->states_usage[last_idx].disable ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) drv->states[last_idx].exit_latency_ns > latency_req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) for (i = last_idx - 1; i > first_idx; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (drv->states[i].exit_latency_ns <= latency_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) ladder_do_selection(dev, ldev, last_idx, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (last_idx > first_idx &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) last_residency < last_state->threshold.demotion_time_ns) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) last_state->stats.demotion_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) last_state->stats.promotion_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ladder_do_selection(dev, ldev, last_idx, last_idx - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return last_idx - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /* otherwise remain at the current state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) return last_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * ladder_enable_device - setup for the governor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * @drv: cpuidle driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * @dev: the CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) static int ladder_enable_device(struct cpuidle_driver *drv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct cpuidle_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct ladder_device *ldev = &per_cpu(ladder_devices, dev->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) struct ladder_device_state *lstate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct cpuidle_state *state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) dev->last_state_idx = first_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) for (i = first_idx; i < drv->state_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) state = &drv->states[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) lstate = &ldev->states[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) lstate->stats.promotion_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) lstate->stats.demotion_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) lstate->threshold.promotion_count = PROMOTION_COUNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) lstate->threshold.demotion_count = DEMOTION_COUNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) if (i < drv->state_count - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) lstate->threshold.promotion_time_ns = state->exit_latency_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if (i > first_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) lstate->threshold.demotion_time_ns = state->exit_latency_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * ladder_reflect - update the correct last_state_idx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * @dev: the CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * @index: the index of actual state entered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static void ladder_reflect(struct cpuidle_device *dev, int index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (index > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) dev->last_state_idx = index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static struct cpuidle_governor ladder_governor = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) .name = "ladder",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) .rating = 10,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) .enable = ladder_enable_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) .select = ladder_select_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) .reflect = ladder_reflect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * init_ladder - initializes the governor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static int __init init_ladder(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * When NO_HZ is disabled, or when booting with nohz=off, the ladder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * governor is better so give it a higher rating than the menu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * governor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (!tick_nohz_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) ladder_governor.rating = 25;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return cpuidle_register_governor(&ladder_governor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) postcore_initcall(init_ladder);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags