^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright 2018 Linaro Limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * The idle injection framework provides a way to force CPUs to enter idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * states for a specified fraction of time over a specified period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * It relies on the smpboot kthreads feature providing common code for CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * hotplug and thread [un]parking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * All of the kthreads used for idle injection are created at init time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Next, the users of the the idle injection framework provide a cpumask via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * its register function. The kthreads will be synchronized with respect to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * this cpumask.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * The idle + run duration is specified via separate helpers and that allows
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * idle injection to be started.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * The idle injection kthreads will call play_idle_precise() with the idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * duration and max allowed latency specified as per the above.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * After all of them have been woken up, a timer is set to start the next idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * injection cycle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * The timer interrupt handler will wake up the idle injection kthreads for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * all of the CPUs in the cpumask provided by the user.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * Idle injection is stopped synchronously and no leftover idle injection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * kthread activity after its completion is guaranteed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * It is up to the user of this framework to provide a lock for higher-level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * synchronization to prevent race conditions like starting idle injection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * while unregistering from the framework.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define pr_fmt(fmt) "ii_dev: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/hrtimer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/smpboot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/idle_inject.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <uapi/linux/sched/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * struct idle_inject_thread - task on/off switch structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * @tsk: task injecting the idle cycles
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * @should_run: whether or not to run the task (for the smpboot kthread API)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct idle_inject_thread {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct task_struct *tsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) int should_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * struct idle_inject_device - idle injection data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * @timer: idle injection period timer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * @idle_duration_us: duration of CPU idle time to inject
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * @run_duration_us: duration of CPU run time to allow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * @latency_us: max allowed latency
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * @cpumask: mask of CPUs affected by idle injection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct idle_inject_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct hrtimer timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) unsigned int idle_duration_us;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) unsigned int run_duration_us;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) unsigned int latency_us;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) unsigned long cpumask[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * idle_inject_wakeup - Wake up idle injection threads
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * @ii_dev: target idle injection device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * Every idle injection task associated with the given idle injection device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * and running on an online CPU will be woken up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct idle_inject_thread *iit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) iit = per_cpu_ptr(&idle_inject_thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) iit->should_run = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) wake_up_process(iit->tsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * idle_inject_timer_fn - idle injection timer function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * @timer: idle injection hrtimer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * This function is called when the idle injection timer expires. It wakes up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * idle injection tasks associated with the timer and they, in turn, invoke
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * play_idle_precise() to inject a specified amount of CPU idle time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * Return: HRTIMER_RESTART.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) unsigned int duration_us;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct idle_inject_device *ii_dev =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) container_of(timer, struct idle_inject_device, timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) duration_us = READ_ONCE(ii_dev->run_duration_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) duration_us += READ_ONCE(ii_dev->idle_duration_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) idle_inject_wakeup(ii_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) return HRTIMER_RESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * idle_inject_fn - idle injection work function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * @cpu: the CPU owning the task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * This function calls play_idle_precise() to inject a specified amount of CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * idle time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) static void idle_inject_fn(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct idle_inject_device *ii_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) struct idle_inject_thread *iit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) ii_dev = per_cpu(idle_inject_device, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) iit = per_cpu_ptr(&idle_inject_thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * Let the smpboot main loop know that the task should not run again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) iit->should_run = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * idle_inject_set_duration - idle and run duration update helper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * @run_duration_us: CPU run time to allow in microseconds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * @idle_duration_us: CPU idle time to inject in microseconds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) void idle_inject_set_duration(struct idle_inject_device *ii_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) unsigned int run_duration_us,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) unsigned int idle_duration_us)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (run_duration_us && idle_duration_us) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * idle_inject_get_duration - idle and run duration retrieval helper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * @run_duration_us: memory location to store the current CPU run time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * @idle_duration_us: memory location to store the current CPU idle time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) void idle_inject_get_duration(struct idle_inject_device *ii_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) unsigned int *run_duration_us,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) unsigned int *idle_duration_us)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) *run_duration_us = READ_ONCE(ii_dev->run_duration_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * idle_inject_set_latency - set the maximum latency allowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * @latency_us: set the latency requirement for the idle state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) void idle_inject_set_latency(struct idle_inject_device *ii_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) unsigned int latency_us)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) WRITE_ONCE(ii_dev->latency_us, latency_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * idle_inject_start - start idle injections
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * @ii_dev: idle injection control device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * The function starts idle injection by first waking up all of the idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * injection kthreads associated with @ii_dev to let them inject CPU idle time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * sets up a timer to start the next idle injection period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) int idle_inject_start(struct idle_inject_device *ii_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (!idle_duration_us || !run_duration_us)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) idle_inject_wakeup(ii_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) hrtimer_start(&ii_dev->timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) ns_to_ktime((idle_duration_us + run_duration_us) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) NSEC_PER_USEC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * idle_inject_stop - stops idle injections
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * @ii_dev: idle injection control device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * The function stops idle injection and waits for the threads to finish work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * If CPU idle time is being injected when this function runs, then it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * wait until the end of the cycle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * When it returns, there is no more idle injection kthread activity. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * kthreads are scheduled out and the periodic timer is off.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) void idle_inject_stop(struct idle_inject_device *ii_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct idle_inject_thread *iit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) hrtimer_cancel(&ii_dev->timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * Stopping idle injection requires all of the idle injection kthreads
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * associated with the given cpumask to be parked and stay that way, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * prevent CPUs from going online at this point. Any CPUs going online
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * after the loop below will be covered by clearing the should_run flag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * that will cause the smpboot main loop to schedule them out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) cpu_hotplug_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * Iterate over all (online + offline) CPUs here in case one of them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * goes offline with the should_run flag set so as to prevent its idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * injection kthread from running when the CPU goes online again after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * the ii_dev has been freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) iit = per_cpu_ptr(&idle_inject_thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) iit->should_run = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) wait_task_inactive(iit->tsk, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) cpu_hotplug_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * idle_inject_setup - prepare the current task for idle injection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * @cpu: not used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * Called once, this function is in charge of setting the current task's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * scheduler parameters to make it an RT task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) static void idle_inject_setup(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) sched_set_fifo(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * idle_inject_should_run - function helper for the smpboot API
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * @cpu: CPU the kthread is running on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * Return: whether or not the thread can run.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static int idle_inject_should_run(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct idle_inject_thread *iit =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) per_cpu_ptr(&idle_inject_thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return iit->should_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * idle_inject_register - initialize idle injection on a set of CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * @cpumask: CPUs to be affected by idle injection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * This function creates an idle injection control device structure for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * given set of CPUs and initializes the timer associated with it. It does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * start any injection cycles.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * Return: NULL if memory allocation fails, idle injection control device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * pointer on success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct idle_inject_device *ii_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) int cpu, cpu_rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (!ii_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) ii_dev->timer.function = idle_inject_timer_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) ii_dev->latency_us = UINT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (per_cpu(idle_inject_device, cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) pr_err("cpu%d is already registered\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) goto out_rollback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) per_cpu(idle_inject_device, cpu) = ii_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) return ii_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) out_rollback:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (cpu == cpu_rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) per_cpu(idle_inject_device, cpu_rb) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) kfree(ii_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * idle_inject_unregister - unregister idle injection control device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * @ii_dev: idle injection control device to unregister
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * The function stops idle injection for the given control device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * unregisters its kthreads and frees memory allocated when that device was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * created.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) void idle_inject_unregister(struct idle_inject_device *ii_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) idle_inject_stop(ii_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) per_cpu(idle_inject_device, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) kfree(ii_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static struct smp_hotplug_thread idle_inject_threads = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) .store = &idle_inject_thread.tsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) .setup = idle_inject_setup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) .thread_fn = idle_inject_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) .thread_comm = "idle_inject/%u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) .thread_should_run = idle_inject_should_run,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) static int __init idle_inject_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return smpboot_register_percpu_thread(&idle_inject_threads);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) early_initcall(idle_inject_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags