^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * intel_powerclamp.c - package c-state idle injection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2012, Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Authors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Arjan van de Ven <arjan@linux.intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Jacob Pan <jacob.jun.pan@linux.intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * TODO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * 1. better handle wakeup from external interrupts, currently a fixed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * compensation is added to clamping duration when excessive amount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * of wakeups are observed during idle time. the reason is that in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * case of external interrupts without need for ack, clamping down
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * cpu in non-irq context does not reduce irq. for majority of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * cases, clamping down cpu does help reduce irq as well, we should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * be able to differentiate the two cases and give a quantitative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * solution for the irqs that we can control. perhaps based on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * get_cpu_iowait_time_us()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * 2. synchronization with other hw blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/thermal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/tick.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/sched/rt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <uapi/linux/sched/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <asm/msr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <asm/mwait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/cpu_device_id.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <asm/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define MAX_TARGET_RATIO (50U)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /* For each undisturbed clamping period (no extra wake ups during idle time),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * we increment the confidence counter for the given target ratio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * CONFIDENCE_OK defines the level where runtime calibration results are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * valid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define CONFIDENCE_OK (3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /* Default idle injection duration, driver adjust sleep time to meet target
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * idle ratio. Similar to frequency modulation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define DEFAULT_DURATION_JIFFIES (6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static unsigned int target_mwait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static struct dentry *debug_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) /* user selected target */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) static unsigned int set_target_ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static unsigned int current_ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static bool should_skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) static bool reduce_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static atomic_t idle_wakeup_counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static unsigned int control_cpu; /* The cpu assigned to collect stat and update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * control parameters. default to BSP but BSP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * can be offlined.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static bool clamping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct powerclamp_worker_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct kthread_worker *worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct kthread_work balancing_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct kthread_delayed_work idle_injection_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) unsigned int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) unsigned int guard;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) unsigned int window_size_now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) unsigned int target_ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) unsigned int duration_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) bool clamping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) static struct powerclamp_worker_data __percpu *worker_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static struct thermal_cooling_device *cooling_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static unsigned long *cpu_clamping_mask; /* bit map for tracking per cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * clamping kthread worker
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static unsigned int duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static unsigned int pkg_cstate_ratio_cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static unsigned int window_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static int duration_set(const char *arg, const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) unsigned long new_duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) ret = kstrtoul(arg, 10, &new_duration);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (new_duration > 25 || new_duration < 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) pr_err("Out of recommended range %lu, between 6-25ms\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) new_duration);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) duration = clamp(new_duration, 6ul, 25ul);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) static const struct kernel_param_ops duration_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) .set = duration_set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) .get = param_get_int,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) module_param_cb(duration, &duration_ops, &duration, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) MODULE_PARM_DESC(duration, "forced idle time for each attempt in msec.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) struct powerclamp_calibration_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) unsigned long confidence; /* used for calibration, basically a counter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * gets incremented each time a clamping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * period is completed without extra wakeups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * once that counter is reached given level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * compensation is deemed usable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) unsigned long steady_comp; /* steady state compensation used when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * no extra wakeups occurred.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) unsigned long dynamic_comp; /* compensate excessive wakeup from idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * mostly from external interrupts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static struct powerclamp_calibration_data cal_data[MAX_TARGET_RATIO];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) static int window_size_set(const char *arg, const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) unsigned long new_window_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) ret = kstrtoul(arg, 10, &new_window_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) goto exit_win;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (new_window_size > 10 || new_window_size < 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) pr_err("Out of recommended window size %lu, between 2-10\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) new_window_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) window_size = clamp(new_window_size, 2ul, 10ul);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) exit_win:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static const struct kernel_param_ops window_size_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) .set = window_size_set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) .get = param_get_int,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) module_param_cb(window_size, &window_size_ops, &window_size, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) MODULE_PARM_DESC(window_size, "sliding window in number of clamping cycles\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) "\tpowerclamp controls idle ratio within this window. larger\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) "\twindow size results in slower response time but more smooth\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) "\tclamping results. default to 2.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) static void find_target_mwait(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) unsigned int eax, ebx, ecx, edx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) unsigned int highest_cstate = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) unsigned int highest_subcstate = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) edx >>= MWAIT_SUBSTATE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (edx & MWAIT_SUBSTATE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) highest_cstate = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) target_mwait = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) (highest_subcstate - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) struct pkg_cstate_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) bool skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) int msr_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) int cstate_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) #define PKG_CSTATE_INIT(id) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) .msr_index = MSR_PKG_C##id##_RESIDENCY, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) .cstate_id = id \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static struct pkg_cstate_info pkg_cstates[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) PKG_CSTATE_INIT(2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) PKG_CSTATE_INIT(3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) PKG_CSTATE_INIT(6),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) PKG_CSTATE_INIT(7),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) PKG_CSTATE_INIT(8),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) PKG_CSTATE_INIT(9),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) PKG_CSTATE_INIT(10),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {NULL},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static bool has_pkg_state_counter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct pkg_cstate_info *info = pkg_cstates;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /* check if any one of the counter msrs exists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) while (info->msr_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (!rdmsrl_safe(info->msr_index, &val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) info++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) static u64 pkg_state_counter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) u64 count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) struct pkg_cstate_info *info = pkg_cstates;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) while (info->msr_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (!info->skip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (!rdmsrl_safe(info->msr_index, &val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) count += val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) info->skip = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) info++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) static unsigned int get_compensation(int ratio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) unsigned int comp = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) /* we only use compensation if all adjacent ones are good */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) if (ratio == 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) cal_data[ratio].confidence >= CONFIDENCE_OK &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) cal_data[ratio + 1].confidence >= CONFIDENCE_OK &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) cal_data[ratio + 2].confidence >= CONFIDENCE_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) comp = (cal_data[ratio].steady_comp +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) cal_data[ratio + 1].steady_comp +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) cal_data[ratio + 2].steady_comp) / 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) } else if (ratio == MAX_TARGET_RATIO - 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) cal_data[ratio].confidence >= CONFIDENCE_OK &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) cal_data[ratio - 2].confidence >= CONFIDENCE_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) comp = (cal_data[ratio].steady_comp +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) cal_data[ratio - 1].steady_comp +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) cal_data[ratio - 2].steady_comp) / 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) } else if (cal_data[ratio].confidence >= CONFIDENCE_OK &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) cal_data[ratio + 1].confidence >= CONFIDENCE_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) comp = (cal_data[ratio].steady_comp +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) cal_data[ratio - 1].steady_comp +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) cal_data[ratio + 1].steady_comp) / 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) /* REVISIT: simple penalty of double idle injection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) if (reduce_irq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) comp = ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) /* do not exceed limit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (comp + ratio >= MAX_TARGET_RATIO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) comp = MAX_TARGET_RATIO - ratio - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return comp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) static void adjust_compensation(int target_ratio, unsigned int win)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) int delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) struct powerclamp_calibration_data *d = &cal_data[target_ratio];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * adjust compensations if confidence level has not been reached or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * there are too many wakeups during the last idle injection period, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * cannot trust the data for compensation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (d->confidence >= CONFIDENCE_OK ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) atomic_read(&idle_wakeup_counter) >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) win * num_online_cpus())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) delta = set_target_ratio - current_ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /* filter out bad data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (delta >= 0 && delta <= (1+target_ratio/10)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (d->steady_comp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) d->steady_comp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) roundup(delta+d->steady_comp, 2)/2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) d->steady_comp = delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) d->confidence++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) static bool powerclamp_adjust_controls(unsigned int target_ratio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) unsigned int guard, unsigned int win)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) static u64 msr_last, tsc_last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) u64 msr_now, tsc_now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) u64 val64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /* check result for the last window */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) msr_now = pkg_state_counter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) tsc_now = rdtsc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /* calculate pkg cstate vs tsc ratio */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) if (!msr_last || !tsc_last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) current_ratio = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) else if (tsc_now-tsc_last) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) val64 = 100*(msr_now-msr_last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) do_div(val64, (tsc_now-tsc_last));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) current_ratio = val64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) /* update record */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) msr_last = msr_now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) tsc_last = tsc_now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) adjust_compensation(target_ratio, win);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * too many external interrupts, set flag such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * that we can take measure later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) reduce_irq = atomic_read(&idle_wakeup_counter) >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 2 * win * num_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) atomic_set(&idle_wakeup_counter, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /* if we are above target+guard, skip */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) return set_target_ratio + guard <= current_ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) static void clamp_balancing_func(struct kthread_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) struct powerclamp_worker_data *w_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) int sleeptime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) unsigned long target_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) unsigned int compensated_ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) int interval; /* jiffies to sleep for each attempt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) w_data = container_of(work, struct powerclamp_worker_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) balancing_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * make sure user selected ratio does not take effect until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * the next round. adjust target_ratio if user has changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * target such that we can converge quickly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) w_data->target_ratio = READ_ONCE(set_target_ratio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) w_data->guard = 1 + w_data->target_ratio / 20;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) w_data->window_size_now = window_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) w_data->duration_jiffies = msecs_to_jiffies(duration);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) w_data->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * systems may have different ability to enter package level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * c-states, thus we need to compensate the injected idle ratio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * to achieve the actual target reported by the HW.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) compensated_ratio = w_data->target_ratio +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) get_compensation(w_data->target_ratio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (compensated_ratio <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) compensated_ratio = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) interval = w_data->duration_jiffies * 100 / compensated_ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) /* align idle time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) target_jiffies = roundup(jiffies, interval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) sleeptime = target_jiffies - jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (sleeptime <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) sleeptime = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (clamping && w_data->clamping && cpu_online(w_data->cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) kthread_queue_delayed_work(w_data->worker,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) &w_data->idle_injection_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) sleeptime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static void clamp_idle_injection_func(struct kthread_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) struct powerclamp_worker_data *w_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) w_data = container_of(work, struct powerclamp_worker_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) idle_injection_work.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * only elected controlling cpu can collect stats and update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * control parameters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) if (w_data->cpu == control_cpu &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) !(w_data->count % w_data->window_size_now)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) should_skip =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) powerclamp_adjust_controls(w_data->target_ratio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) w_data->guard,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) w_data->window_size_now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (should_skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) goto balance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) play_idle(jiffies_to_usecs(w_data->duration_jiffies));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) balance:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) if (clamping && w_data->clamping && cpu_online(w_data->cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) kthread_queue_work(w_data->worker, &w_data->balancing_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) * 1 HZ polling while clamping is active, useful for userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * to monitor actual idle ratio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) static void poll_pkg_cstate(struct work_struct *dummy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) static DECLARE_DELAYED_WORK(poll_pkg_cstate_work, poll_pkg_cstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) static void poll_pkg_cstate(struct work_struct *dummy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) static u64 msr_last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) static u64 tsc_last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) u64 msr_now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) u64 tsc_now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) u64 val64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) msr_now = pkg_state_counter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) tsc_now = rdtsc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) /* calculate pkg cstate vs tsc ratio */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (!msr_last || !tsc_last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) pkg_cstate_ratio_cur = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (tsc_now - tsc_last) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) val64 = 100 * (msr_now - msr_last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) do_div(val64, (tsc_now - tsc_last));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) pkg_cstate_ratio_cur = val64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) /* update record */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) msr_last = msr_now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) tsc_last = tsc_now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (true == clamping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) schedule_delayed_work(&poll_pkg_cstate_work, HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) static void start_power_clamp_worker(unsigned long cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) struct kthread_worker *worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) worker = kthread_create_worker_on_cpu(cpu, 0, "kidle_inj/%ld", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if (IS_ERR(worker))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) w_data->worker = worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) w_data->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) w_data->cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) w_data->clamping = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) set_bit(cpu, cpu_clamping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) sched_set_fifo(worker->task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) kthread_init_work(&w_data->balancing_work, clamp_balancing_func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) kthread_init_delayed_work(&w_data->idle_injection_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) clamp_idle_injection_func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) kthread_queue_work(w_data->worker, &w_data->balancing_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) static void stop_power_clamp_worker(unsigned long cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (!w_data->worker)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) w_data->clamping = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * Make sure that all works that get queued after this point see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * the clamping disabled. The counter part is not needed because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * there is an implicit memory barrier when the queued work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * is proceed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) kthread_cancel_work_sync(&w_data->balancing_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) kthread_cancel_delayed_work_sync(&w_data->idle_injection_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * The balancing work still might be queued here because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * the handling of the "clapming" variable, cancel, and queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * operations are not synchronized via a lock. But it is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) * a big deal. The balancing work is fast and destroy kthread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) * will wait for it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) clear_bit(w_data->cpu, cpu_clamping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) kthread_destroy_worker(w_data->worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) w_data->worker = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) static int start_power_clamp(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) unsigned long cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /* prevent cpu hotplug */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) get_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) /* prefer BSP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) control_cpu = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (!cpu_online(control_cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) control_cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) clamping = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) schedule_delayed_work(&poll_pkg_cstate_work, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) /* start one kthread worker per online cpu */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) start_power_clamp_worker(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) put_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) static void end_power_clamp(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * Block requeuing in all the kthread workers. They will flush and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * stop faster.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) clamping = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) if (bitmap_weight(cpu_clamping_mask, num_possible_cpus())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) for_each_set_bit(i, cpu_clamping_mask, num_possible_cpus()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) pr_debug("clamping worker for cpu %d alive, destroy\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) stop_power_clamp_worker(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) static int powerclamp_cpu_online(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) if (clamping == false)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) start_power_clamp_worker(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) /* prefer BSP as controlling CPU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) if (cpu == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) control_cpu = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) static int powerclamp_cpu_predown(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (clamping == false)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) stop_power_clamp_worker(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (cpu != control_cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) control_cpu = cpumask_first(cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) if (control_cpu == cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) control_cpu = cpumask_next(cpu, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) static int powerclamp_get_max_state(struct thermal_cooling_device *cdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) unsigned long *state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) *state = MAX_TARGET_RATIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) static int powerclamp_get_cur_state(struct thermal_cooling_device *cdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) unsigned long *state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (true == clamping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) *state = pkg_cstate_ratio_cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* to save power, do not poll idle ratio while not clamping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) *state = -1; /* indicates invalid state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) static int powerclamp_set_cur_state(struct thermal_cooling_device *cdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) unsigned long new_target_ratio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) new_target_ratio = clamp(new_target_ratio, 0UL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) (unsigned long) (MAX_TARGET_RATIO-1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (set_target_ratio == 0 && new_target_ratio > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) pr_info("Start idle injection to reduce power\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) set_target_ratio = new_target_ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) ret = start_power_clamp();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) goto exit_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) } else if (set_target_ratio > 0 && new_target_ratio == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) pr_info("Stop forced idle injection\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) end_power_clamp();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) set_target_ratio = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) } else /* adjust currently running */ {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) set_target_ratio = new_target_ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) /* make new set_target_ratio visible to other cpus */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) exit_set:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) /* bind to generic thermal layer as cooling device*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) static struct thermal_cooling_device_ops powerclamp_cooling_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) .get_max_state = powerclamp_get_max_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) .get_cur_state = powerclamp_get_cur_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) .set_cur_state = powerclamp_set_cur_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) static int __init powerclamp_probe(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) if (!x86_match_cpu(intel_powerclamp_ids)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) pr_err("CPU does not support MWAIT\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) /* The goal for idle time alignment is to achieve package cstate. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) if (!has_pkg_state_counter()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) pr_info("No package C-state available\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) /* find the deepest mwait value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) find_target_mwait();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) static int powerclamp_debug_show(struct seq_file *m, void *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) seq_printf(m, "controlling cpu: %d\n", control_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) seq_printf(m, "pct confidence steady dynamic (compensation)\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) for (i = 0; i < MAX_TARGET_RATIO; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) seq_printf(m, "%d\t%lu\t%lu\t%lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) cal_data[i].confidence,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) cal_data[i].steady_comp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) cal_data[i].dynamic_comp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) DEFINE_SHOW_ATTRIBUTE(powerclamp_debug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) static inline void powerclamp_create_debug_files(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) debug_dir = debugfs_create_dir("intel_powerclamp", NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) debugfs_create_file("powerclamp_calib", S_IRUGO, debug_dir, cal_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) &powerclamp_debug_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) static enum cpuhp_state hp_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) static int __init powerclamp_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) int retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) int bitmap_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) bitmap_size = BITS_TO_LONGS(num_possible_cpus()) * sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) cpu_clamping_mask = kzalloc(bitmap_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) if (!cpu_clamping_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) /* probe cpu features and ids here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) retval = powerclamp_probe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) if (retval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) goto exit_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) /* set default limit, maybe adjusted during runtime based on feedback */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) window_size = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) "thermal/intel_powerclamp:online",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) powerclamp_cpu_online,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) powerclamp_cpu_predown);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (retval < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) goto exit_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) hp_state = retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) worker_data = alloc_percpu(struct powerclamp_worker_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) if (!worker_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) retval = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) goto exit_unregister;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) cooling_dev = thermal_cooling_device_register("intel_powerclamp", NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) &powerclamp_cooling_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) if (IS_ERR(cooling_dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) retval = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) goto exit_free_thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) if (!duration)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) duration = jiffies_to_msecs(DEFAULT_DURATION_JIFFIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) powerclamp_create_debug_files();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) exit_free_thread:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) free_percpu(worker_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) exit_unregister:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) cpuhp_remove_state_nocalls(hp_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) exit_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) kfree(cpu_clamping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) return retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) module_init(powerclamp_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) static void __exit powerclamp_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) end_power_clamp();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) cpuhp_remove_state_nocalls(hp_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) free_percpu(worker_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) thermal_cooling_device_unregister(cooling_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) kfree(cpu_clamping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) cancel_delayed_work_sync(&poll_pkg_cstate_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) debugfs_remove_recursive(debug_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) module_exit(powerclamp_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) MODULE_AUTHOR("Arjan van de Ven <arjan@linux.intel.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@linux.intel.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) MODULE_DESCRIPTION("Package Level C-state Idle Injection for Intel CPUs");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate