Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *  Housekeeping management. Manage the targets for routine code that can run on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include "sched.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) EXPORT_SYMBOL_GPL(housekeeping_overridden);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) static cpumask_var_t housekeeping_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) static unsigned int housekeeping_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) bool housekeeping_enabled(enum hk_flags flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 	return !!(housekeeping_flags & flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) EXPORT_SYMBOL_GPL(housekeeping_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) int housekeeping_any_cpu(enum hk_flags flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	if (static_branch_unlikely(&housekeeping_overridden)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 		if (housekeeping_flags & flags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 			cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 			if (cpu < nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 				return cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 			return cpumask_any_and(housekeeping_mask, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	return smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	if (static_branch_unlikely(&housekeeping_overridden))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 		if (housekeeping_flags & flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 			return housekeeping_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	return cpu_possible_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) EXPORT_SYMBOL_GPL(housekeeping_cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	if (static_branch_unlikely(&housekeeping_overridden))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 		if (housekeeping_flags & flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 			set_cpus_allowed_ptr(t, housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) EXPORT_SYMBOL_GPL(housekeeping_affine);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	if (static_branch_unlikely(&housekeeping_overridden))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 		if (housekeeping_flags & flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 			return cpumask_test_cpu(cpu, housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) void __init housekeeping_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	if (!housekeeping_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	static_branch_enable(&housekeeping_overridden);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	if (housekeeping_flags & HK_FLAG_TICK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 		sched_tick_offload_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	/* We need at least one CPU to handle housekeeping work */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) static int __init housekeeping_setup(char *str, enum hk_flags flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	cpumask_var_t non_housekeeping_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	cpumask_var_t tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	alloc_bootmem_cpumask_var(&non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	err = cpulist_parse(str, non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 		free_bootmem_cpumask_var(non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	alloc_bootmem_cpumask_var(&tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	if (!housekeeping_flags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 		alloc_bootmem_cpumask_var(&housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 		cpumask_andnot(housekeeping_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 			       cpu_possible_mask, non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 		if (cpumask_empty(tmp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 			pr_warn("Housekeeping: must include one present CPU, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 				"using boot CPU:%d\n", smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 			__cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 			__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 		cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 		if (cpumask_empty(tmp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 			__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 		cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 		if (!cpumask_equal(tmp, housekeeping_mask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 			pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 			free_bootmem_cpumask_var(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 			free_bootmem_cpumask_var(non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 			return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	free_bootmem_cpumask_var(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 		if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 			tick_nohz_full_setup(non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 		} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 			pr_warn("Housekeeping: nohz unsupported."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 				" Build with CONFIG_NO_HZ_FULL\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 			free_bootmem_cpumask_var(non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 			return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	housekeeping_flags |= flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	free_bootmem_cpumask_var(non_housekeeping_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static int __init housekeeping_nohz_full_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	unsigned int flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 		HK_FLAG_MISC | HK_FLAG_KTHREAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	return housekeeping_setup(str, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) __setup("nohz_full=", housekeeping_nohz_full_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static int __init housekeeping_isolcpus_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	unsigned int flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	bool illegal = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	char *par;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	while (isalpha(*str)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 		if (!strncmp(str, "nohz,", 5)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 			str += 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 			flags |= HK_FLAG_TICK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 		if (!strncmp(str, "domain,", 7)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 			str += 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 			flags |= HK_FLAG_DOMAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 		if (!strncmp(str, "managed_irq,", 12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 			str += 12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 			flags |= HK_FLAG_MANAGED_IRQ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 		 * Skip unknown sub-parameter and validate that it is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 		 * containing an invalid character.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 		for (par = str, len = 0; *str && *str != ','; str++, len++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 			if (!isalpha(*str) && *str != '_')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 				illegal = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 		if (illegal) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 			pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 			return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 		pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		str++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	/* Default behaviour for isolcpus without flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	if (!flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 		flags |= HK_FLAG_DOMAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	return housekeeping_setup(str, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) __setup("isolcpus=", housekeeping_isolcpus_setup);