Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #include "sched-pelt.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) int __update_load_avg_blocked_se(u64 now, struct sched_entity *se);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) int update_dl_rq_load_avg(u64 now, struct rq *rq, int running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #ifdef CONFIG_SCHED_THERMAL_PRESSURE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) static inline u64 thermal_load_avg(struct rq *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 	return READ_ONCE(rq->avg_thermal.load_avg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) static inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) static inline u64 thermal_load_avg(struct rq *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) int update_irq_load_avg(struct rq *rq, u64 running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) static inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) update_irq_load_avg(struct rq *rq, u64 running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) #define PELT_MIN_DIVIDER	(LOAD_AVG_MAX - 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) static inline u32 get_pelt_divider(struct sched_avg *avg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	return PELT_MIN_DIVIDER + avg->period_contrib;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) static inline void cfs_se_util_change(struct sched_avg *avg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	unsigned int enqueued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	if (!sched_feat(UTIL_EST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	/* Avoid store if the flag has been already reset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	enqueued = avg->util_est.enqueued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	if (!(enqueued & UTIL_AVG_UNCHANGED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	/* Reset flag to report util_avg has been updated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	enqueued &= ~UTIL_AVG_UNCHANGED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	WRITE_ONCE(avg->util_est.enqueued, enqueued);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65)  * The clock_pelt scales the time to reflect the effective amount of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66)  * computation done during the running delta time but then sync back to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67)  * clock_task when rq is idle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70)  * absolute time   | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71)  * @ max capacity  ------******---------------******---------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72)  * @ half capacity ------************---------************---------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73)  * clock pelt      | 1| 2|    3|    4| 7| 8| 9|   10|   11|14|15|16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) static inline void update_rq_clock_pelt(struct rq *rq, s64 delta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	if (unlikely(is_idle_task(rq->curr))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 		/* The rq is idle, we can sync to clock_task */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 		rq->clock_pelt  = rq_clock_task(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	 * When a rq runs at a lower compute capacity, it will need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	 * more time to do the same amount of work than at max
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	 * capacity. In order to be invariant, we scale the delta to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	 * reflect how much work has been really done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	 * Running longer results in stealing idle time that will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	 * disturb the load signal compared to max capacity. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	 * stolen idle time will be automatically reflected when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	 * rq will be idle and the clock will be synced with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	 * rq_clock_task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	 * Scale the elapsed time to reflect the real amount of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	 * computation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	rq->clock_pelt += delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)  * When rq becomes idle, we have to check if it has lost idle time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)  * because it was fully busy. A rq is fully used when the /Sum util_sum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)  * is greater or equal to:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)  * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)  * For optimization and computing rounding purpose, we don't take into account
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)  * the position in the current window (period_contrib) and we use the higher
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)  * bound of util_sum to decide.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) static inline void update_idle_rq_clock_pelt(struct rq *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	u32 util_sum = rq->cfs.avg.util_sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	util_sum += rq->avg_rt.util_sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	util_sum += rq->avg_dl.util_sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	 * Reflecting stolen time makes sense only if the idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	 * phase would be present at max capacity. As soon as the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	 * utilization of a rq has reached the maximum value, it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	 * considered as an always runnig rq without idle time to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	 * steal. This potential idle time is considered as lost in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	 * this case. We keep track of this lost idle time compare to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	 * rq's clock_task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	if (util_sum >= divider)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 		rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static inline u64 rq_clock_pelt(struct rq *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	lockdep_assert_held(&rq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	assert_clock_updated(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	return rq->clock_pelt - rq->lost_idle_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #ifdef CONFIG_CFS_BANDWIDTH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) /* rq->task_clock normalized against any time this cfs_rq has spent throttled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	if (unlikely(cfs_rq->throttle_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 		return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	return rq_clock_pelt(rq_of(cfs_rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) static inline u64 thermal_load_avg(struct rq *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) static inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) update_irq_load_avg(struct rq *rq, u64 running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static inline u64 rq_clock_pelt(struct rq *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	return rq_clock_task(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) update_rq_clock_pelt(struct rq *rq, s64 delta) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) update_idle_rq_clock_pelt(struct rq *rq) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)