Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *  kernel/sched/cpudl.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *  Global CPU deadline management
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *  Author: Juri Lelli <j.lelli@sssup.it>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include "sched.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) static inline int parent(int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 	return (i - 1) >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) static inline int left_child(int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 	return (i << 1) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) static inline int right_child(int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	return (i << 1) + 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) static void cpudl_heapify_down(struct cpudl *cp, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	int l, r, largest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	int orig_cpu = cp->elements[idx].cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	u64 orig_dl = cp->elements[idx].dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	if (left_child(idx) >= cp->size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	/* adapted from lib/prio_heap.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 		u64 largest_dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 		l = left_child(idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 		r = right_child(idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 		largest = idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 		largest_dl = orig_dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 		if ((l < cp->size) && dl_time_before(orig_dl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 						cp->elements[l].dl)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 			largest = l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 			largest_dl = cp->elements[l].dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 		if ((r < cp->size) && dl_time_before(largest_dl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 						cp->elements[r].dl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 			largest = r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 		if (largest == idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 		/* pull largest child onto idx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 		cp->elements[idx].cpu = cp->elements[largest].cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 		cp->elements[idx].dl = cp->elements[largest].dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 		cp->elements[cp->elements[idx].cpu].idx = idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 		idx = largest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	/* actual push down of saved original values orig_* */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	cp->elements[idx].cpu = orig_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	cp->elements[idx].dl = orig_dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	cp->elements[cp->elements[idx].cpu].idx = idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) static void cpudl_heapify_up(struct cpudl *cp, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	int p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	int orig_cpu = cp->elements[idx].cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	u64 orig_dl = cp->elements[idx].dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	if (idx == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 		p = parent(idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 		if (dl_time_before(orig_dl, cp->elements[p].dl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 		/* pull parent onto idx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 		cp->elements[idx].cpu = cp->elements[p].cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 		cp->elements[idx].dl = cp->elements[p].dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		cp->elements[cp->elements[idx].cpu].idx = idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 		idx = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	} while (idx != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	/* actual push up of saved original values orig_* */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	cp->elements[idx].cpu = orig_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	cp->elements[idx].dl = orig_dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	cp->elements[cp->elements[idx].cpu].idx = idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) static void cpudl_heapify(struct cpudl *cp, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	if (idx > 0 && dl_time_before(cp->elements[parent(idx)].dl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 				cp->elements[idx].dl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 		cpudl_heapify_up(cp, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 		cpudl_heapify_down(cp, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) static inline int cpudl_maximum(struct cpudl *cp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	return cp->elements[0].cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)  * cpudl_find - find the best (later-dl) CPU in the system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)  * @cp: the cpudl max-heap context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)  * @p: the task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)  * @later_mask: a mask to fill in with the selected CPUs (or NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)  * Returns: int - CPUs were found
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) int cpudl_find(struct cpudl *cp, struct task_struct *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	       struct cpumask *later_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	const struct sched_dl_entity *dl_se = &p->dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	if (later_mask &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	    cpumask_and(later_mask, cp->free_cpus, p->cpus_ptr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 		unsigned long cap, max_cap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 		int cpu, max_cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 		if (!static_branch_unlikely(&sched_asym_cpucapacity))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 			return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 		/* Ensure the capacity of the CPUs fits the task. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		for_each_cpu(cpu, later_mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 			if (!dl_task_fits_capacity(p, cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 				cpumask_clear_cpu(cpu, later_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 				cap = capacity_orig_of(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 				if (cap > max_cap ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 				    (cpu == task_cpu(p) && cap == max_cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 					max_cap = cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 					max_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 				}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 		if (cpumask_empty(later_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 			cpumask_set_cpu(max_cpu, later_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 		return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 		int best_cpu = cpudl_maximum(cp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 		WARN_ON(best_cpu != -1 && !cpu_present(best_cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 		if (cpumask_test_cpu(best_cpu, p->cpus_ptr) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 		    dl_time_before(dl_se->deadline, cp->elements[0].dl)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 			if (later_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 				cpumask_set_cpu(best_cpu, later_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 			return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)  * cpudl_clear - remove a CPU from the cpudl max-heap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)  * @cp: the cpudl max-heap context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)  * @cpu: the target CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)  * Notes: assumes cpu_rq(cpu)->lock is locked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)  * Returns: (void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) void cpudl_clear(struct cpudl *cp, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	int old_idx, new_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	WARN_ON(!cpu_present(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	raw_spin_lock_irqsave(&cp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	old_idx = cp->elements[cpu].idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	if (old_idx == IDX_INVALID) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 		 * Nothing to remove if old_idx was invalid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 		 * This could happen if a rq_offline_dl is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		 * called for a CPU without -dl tasks running.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		new_cpu = cp->elements[cp->size - 1].cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 		cp->elements[old_idx].dl = cp->elements[cp->size - 1].dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 		cp->elements[old_idx].cpu = new_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 		cp->size--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 		cp->elements[new_cpu].idx = old_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 		cp->elements[cpu].idx = IDX_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 		cpudl_heapify(cp, old_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 		cpumask_set_cpu(cpu, cp->free_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	raw_spin_unlock_irqrestore(&cp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)  * cpudl_set - update the cpudl max-heap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)  * @cp: the cpudl max-heap context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)  * @cpu: the target CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)  * @dl: the new earliest deadline for this CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)  * Notes: assumes cpu_rq(cpu)->lock is locked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)  * Returns: (void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) void cpudl_set(struct cpudl *cp, int cpu, u64 dl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	int old_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	WARN_ON(!cpu_present(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	raw_spin_lock_irqsave(&cp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	old_idx = cp->elements[cpu].idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	if (old_idx == IDX_INVALID) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 		int new_idx = cp->size++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 		cp->elements[new_idx].dl = dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 		cp->elements[new_idx].cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 		cp->elements[cpu].idx = new_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 		cpudl_heapify_up(cp, new_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 		cpumask_clear_cpu(cpu, cp->free_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 		cp->elements[old_idx].dl = dl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 		cpudl_heapify(cp, old_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	raw_spin_unlock_irqrestore(&cp->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)  * cpudl_set_freecpu - Set the cpudl.free_cpus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)  * @cp: the cpudl max-heap context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)  * @cpu: rd attached CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) void cpudl_set_freecpu(struct cpudl *cp, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	cpumask_set_cpu(cpu, cp->free_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)  * cpudl_clear_freecpu - Clear the cpudl.free_cpus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)  * @cp: the cpudl max-heap context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)  * @cpu: rd attached CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) void cpudl_clear_freecpu(struct cpudl *cp, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	cpumask_clear_cpu(cpu, cp->free_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)  * cpudl_init - initialize the cpudl structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)  * @cp: the cpudl max-heap context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) int cpudl_init(struct cpudl *cp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	raw_spin_lock_init(&cp->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	cp->size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	cp->elements = kcalloc(nr_cpu_ids,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 			       sizeof(struct cpudl_item),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 			       GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	if (!cp->elements)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 		return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	if (!zalloc_cpumask_var(&cp->free_cpus, GFP_KERNEL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 		kfree(cp->elements);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 		return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	for_each_possible_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 		cp->elements[i].idx = IDX_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)  * cpudl_cleanup - clean up the cpudl structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)  * @cp: the cpudl max-heap context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) void cpudl_cleanup(struct cpudl *cp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	free_cpumask_var(cp->free_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	kfree(cp->elements);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }