^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Scheduler topology setup/handling methods
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include "sched.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <trace/hooks/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) DEFINE_MUTEX(sched_domains_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #ifdef CONFIG_LOCKDEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) EXPORT_SYMBOL_GPL(sched_domains_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) /* Protected by sched_domains_mutex: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) static cpumask_var_t sched_domains_tmpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) static cpumask_var_t sched_domains_tmpmask2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #ifdef CONFIG_SCHED_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) static int __init sched_debug_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) sched_debug_enabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) early_param("sched_debug", sched_debug_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static inline bool sched_debug(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) return sched_debug_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define SD_FLAG(_name, mflags) [__##_name] = { .meta_flags = mflags, .name = #_name },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) const struct sd_flag_debug sd_flag_debug[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/sched/sd_flags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #undef SD_FLAG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct cpumask *groupmask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct sched_group *group = sd->groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) unsigned long flags = sd->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) unsigned int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) cpumask_clear(groupmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) printk(KERN_DEBUG "%*s domain-%d: ", level, "", level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) printk(KERN_CONT "span=%*pbl level=%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) cpumask_pr_args(sched_domain_span(sd)), sd->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) printk(KERN_ERR "ERROR: domain->span does not contain CPU%d\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) if (group && !cpumask_test_cpu(cpu, sched_group_span(group))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) for_each_set_bit(idx, &flags, __SD_FLAG_CNT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) unsigned int flag = BIT(idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) unsigned int meta_flags = sd_flag_debug[idx].meta_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) if ((meta_flags & SDF_SHARED_CHILD) && sd->child &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) !(sd->child->flags & flag))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) printk(KERN_ERR "ERROR: flag %s set here but not in child\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) sd_flag_debug[idx].name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if ((meta_flags & SDF_SHARED_PARENT) && sd->parent &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) !(sd->parent->flags & flag))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) printk(KERN_ERR "ERROR: flag %s set here but not in parent\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) sd_flag_debug[idx].name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) printk(KERN_DEBUG "%*s groups:", level + 1, "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) if (!group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) printk("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) printk(KERN_ERR "ERROR: group is NULL\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (!cpumask_weight(sched_group_span(group))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) printk(KERN_CONT "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) printk(KERN_ERR "ERROR: empty group\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (!(sd->flags & SD_OVERLAP) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) cpumask_intersects(groupmask, sched_group_span(group))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) printk(KERN_CONT "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) printk(KERN_ERR "ERROR: repeated CPUs\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) cpumask_or(groupmask, groupmask, sched_group_span(group));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) printk(KERN_CONT " %d:{ span=%*pbl",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) group->sgc->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) cpumask_pr_args(sched_group_span(group)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) if ((sd->flags & SD_OVERLAP) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) !cpumask_equal(group_balance_mask(group), sched_group_span(group))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) printk(KERN_CONT " mask=%*pbl",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) cpumask_pr_args(group_balance_mask(group)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (group->sgc->capacity != SCHED_CAPACITY_SCALE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) printk(KERN_CONT " cap=%lu", group->sgc->capacity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (group == sd->groups && sd->child &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) !cpumask_equal(sched_domain_span(sd->child),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) sched_group_span(group))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) printk(KERN_ERR "ERROR: domain->groups does not match domain->child\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) printk(KERN_CONT " }");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) group = group->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (group != sd->groups)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) printk(KERN_CONT ",");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) } while (group != sd->groups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) printk(KERN_CONT "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (!cpumask_equal(sched_domain_span(sd), groupmask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) printk(KERN_ERR "ERROR: groups don't span domain->span\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (sd->parent &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) printk(KERN_ERR "ERROR: parent span is not a superset of domain->span\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void sched_domain_debug(struct sched_domain *sd, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) int level = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (!sched_debug_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (!sd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) printk(KERN_DEBUG "CPU%d attaching sched-domain(s):\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) level++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) sd = sd->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (!sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #else /* !CONFIG_SCHED_DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) # define sched_debug_enabled 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) # define sched_domain_debug(sd, cpu) do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static inline bool sched_debug(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #endif /* CONFIG_SCHED_DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /* Generate a mask of SD flags with the SDF_NEEDS_GROUPS metaflag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #define SD_FLAG(name, mflags) (name * !!((mflags) & SDF_NEEDS_GROUPS)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) static const unsigned int SD_DEGENERATE_GROUPS_MASK =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) #include <linux/sched/sd_flags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) #undef SD_FLAG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static int sd_degenerate(struct sched_domain *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (cpumask_weight(sched_domain_span(sd)) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) /* Following flags need at least 2 groups */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) if ((sd->flags & SD_DEGENERATE_GROUPS_MASK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) (sd->groups != sd->groups->next))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /* Following flags don't use groups */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (sd->flags & (SD_WAKE_AFFINE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) unsigned long cflags = sd->flags, pflags = parent->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (sd_degenerate(parent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /* Flags needing groups don't count if only 1 group in parent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (parent->groups == parent->groups->next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) pflags &= ~SD_DEGENERATE_GROUPS_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (~cflags & pflags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) DEFINE_STATIC_KEY_FALSE(sched_energy_present);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) unsigned int sysctl_sched_energy_aware = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) DEFINE_MUTEX(sched_energy_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) bool sched_energy_update;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #ifdef CONFIG_PROC_SYSCTL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) int sched_energy_aware_handler(struct ctl_table *table, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) void *buffer, size_t *lenp, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) int ret, state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (write && !capable(CAP_SYS_ADMIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) if (!ret && write) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) state = static_branch_unlikely(&sched_energy_present);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) if (state != sysctl_sched_energy_aware) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) mutex_lock(&sched_energy_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) sched_energy_update = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) rebuild_sched_domains();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) sched_energy_update = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) mutex_unlock(&sched_energy_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static void free_pd(struct perf_domain *pd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) struct perf_domain *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) while (pd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) tmp = pd->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) kfree(pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) pd = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) static struct perf_domain *find_pd(struct perf_domain *pd, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) while (pd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (cpumask_test_cpu(cpu, perf_domain_span(pd)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) return pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) pd = pd->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) static struct perf_domain *pd_init(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct em_perf_domain *obj = em_cpu_get(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) struct perf_domain *pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (!obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) if (sched_debug())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) pr_info("%s: no EM found for CPU%d\n", __func__, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) pd = kzalloc(sizeof(*pd), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (!pd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) pd->em_pd = obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) static void perf_domain_debug(const struct cpumask *cpu_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) struct perf_domain *pd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (!sched_debug() || !pd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) printk(KERN_DEBUG "root_domain %*pbl:", cpumask_pr_args(cpu_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) while (pd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) printk(KERN_CONT " pd%d:{ cpus=%*pbl nr_pstate=%d }",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) cpumask_first(perf_domain_span(pd)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) cpumask_pr_args(perf_domain_span(pd)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) em_pd_nr_perf_states(pd->em_pd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) pd = pd->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) printk(KERN_CONT "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static void destroy_perf_domain_rcu(struct rcu_head *rp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) struct perf_domain *pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) pd = container_of(rp, struct perf_domain, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) free_pd(pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static void sched_energy_set(bool has_eas)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (!has_eas && static_branch_unlikely(&sched_energy_present)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (sched_debug())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) pr_info("%s: stopping EAS\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static_branch_disable_cpuslocked(&sched_energy_present);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) } else if (has_eas && !static_branch_unlikely(&sched_energy_present)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (sched_debug())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) pr_info("%s: starting EAS\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static_branch_enable_cpuslocked(&sched_energy_present);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * EAS can be used on a root domain if it meets all the following conditions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * 1. an Energy Model (EM) is available;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * 2. the SD_ASYM_CPUCAPACITY flag is set in the sched_domain hierarchy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * 3. no SMT is detected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * 4. the EM complexity is low enough to keep scheduling overheads low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * The complexity of the Energy Model is defined as:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * C = nr_pd * (nr_cpus + nr_ps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * with parameters defined as:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * - nr_pd: the number of performance domains
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * - nr_cpus: the number of CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * - nr_ps: the sum of the number of performance states of all performance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * domains (for example, on a system with 2 performance domains,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * with 10 performance states each, nr_ps = 2 * 10 = 20).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * It is generally not a good idea to use such a model in the wake-up path on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * very complex platforms because of the associated scheduling overheads. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * arbitrary constraint below prevents that. It makes EAS usable up to 16 CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * with per-CPU DVFS and less than 8 performance states each, for example.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) #define EM_MAX_COMPLEXITY 2048
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) static bool build_perf_domains(const struct cpumask *cpu_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) int i, nr_pd = 0, nr_ps = 0, nr_cpus = cpumask_weight(cpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) struct perf_domain *pd = NULL, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) int cpu = cpumask_first(cpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) struct root_domain *rd = cpu_rq(cpu)->rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) bool eas_check = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (!sysctl_sched_energy_aware)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * EAS is enabled for asymmetric CPU capacity topologies.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * Allow vendor to override if desired.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) trace_android_rvh_build_perf_domains(&eas_check);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) if (!per_cpu(sd_asym_cpucapacity, cpu) && !eas_check) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (sched_debug()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) pr_info("rd %*pbl: CPUs do not have asymmetric capacities\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) cpumask_pr_args(cpu_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) /* EAS definitely does *not* handle SMT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (sched_smt_active()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) pr_warn("rd %*pbl: Disabling EAS, SMT is not supported\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) cpumask_pr_args(cpu_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) for_each_cpu(i, cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) /* Skip already covered CPUs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (find_pd(pd, i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) /* Create the new pd and add it to the local list. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) tmp = pd_init(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (!tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) tmp->next = pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) pd = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * Count performance domains and performance states for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) * complexity check.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) nr_pd++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) nr_ps += em_pd_nr_perf_states(pd->em_pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) /* Bail out if the Energy Model complexity is too high. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) if (nr_pd * (nr_ps + nr_cpus) > EM_MAX_COMPLEXITY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) WARN(1, "rd %*pbl: Failed to start EAS, EM complexity is too high\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) cpumask_pr_args(cpu_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) perf_domain_debug(cpu_map, pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) /* Attach the new list of performance domains to the root domain. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) tmp = rd->pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) rcu_assign_pointer(rd->pd, pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) call_rcu(&tmp->rcu, destroy_perf_domain_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) return !!pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) free_pd(pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) tmp = rd->pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) rcu_assign_pointer(rd->pd, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) call_rcu(&tmp->rcu, destroy_perf_domain_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) static void free_pd(struct perf_domain *pd) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) #endif /* CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) static void free_rootdomain(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) cpupri_cleanup(&rd->cpupri);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) cpudl_cleanup(&rd->cpudl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) free_cpumask_var(rd->dlo_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) free_cpumask_var(rd->rto_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) free_cpumask_var(rd->online);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) free_cpumask_var(rd->span);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) free_pd(rd->pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) kfree(rd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) void rq_attach_root(struct rq *rq, struct root_domain *rd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) struct root_domain *old_rd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) raw_spin_lock_irqsave(&rq->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (rq->rd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) old_rd = rq->rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) if (cpumask_test_cpu(rq->cpu, old_rd->online))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) set_rq_offline(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) cpumask_clear_cpu(rq->cpu, old_rd->span);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * If we dont want to free the old_rd yet then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * set old_rd to NULL to skip the freeing later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * in this function:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) if (!atomic_dec_and_test(&old_rd->refcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) old_rd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) atomic_inc(&rd->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) rq->rd = rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) cpumask_set_cpu(rq->cpu, rd->span);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) set_rq_online(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) raw_spin_unlock_irqrestore(&rq->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (old_rd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) call_rcu(&old_rd->rcu, free_rootdomain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) void sched_get_rd(struct root_domain *rd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) atomic_inc(&rd->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) void sched_put_rd(struct root_domain *rd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (!atomic_dec_and_test(&rd->refcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) call_rcu(&rd->rcu, free_rootdomain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) static int init_rootdomain(struct root_domain *rd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (!zalloc_cpumask_var(&rd->span, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (!zalloc_cpumask_var(&rd->online, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) goto free_span;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (!zalloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) goto free_online;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (!zalloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) goto free_dlo_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) #ifdef HAVE_RT_PUSH_IPI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) rd->rto_cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) raw_spin_lock_init(&rd->rto_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) init_irq_work(&rd->rto_push_work, rto_push_irq_work_func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) init_dl_bw(&rd->dl_bw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (cpudl_init(&rd->cpudl) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) goto free_rto_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (cpupri_init(&rd->cpupri) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) goto free_cpudl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) free_cpudl:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) cpudl_cleanup(&rd->cpudl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) free_rto_mask:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) free_cpumask_var(rd->rto_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) free_dlo_mask:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) free_cpumask_var(rd->dlo_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) free_online:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) free_cpumask_var(rd->online);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) free_span:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) free_cpumask_var(rd->span);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * By default the system creates a single root-domain with all CPUs as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) * members (mimicking the global state we have today).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) struct root_domain def_root_domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) void init_defrootdomain(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) init_rootdomain(&def_root_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) atomic_set(&def_root_domain.refcount, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) static struct root_domain *alloc_rootdomain(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) struct root_domain *rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) rd = kzalloc(sizeof(*rd), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (!rd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (init_rootdomain(rd) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) kfree(rd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) return rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) static void free_sched_groups(struct sched_group *sg, int free_sgc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) struct sched_group *tmp, *first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (!sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) first = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) tmp = sg->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) kfree(sg->sgc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) if (atomic_dec_and_test(&sg->ref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) kfree(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) sg = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) } while (sg != first);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) static void destroy_sched_domain(struct sched_domain *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) * A normal sched domain may have multiple group references, an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * overlapping domain, having private groups, only one. Iterate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * dropping group/capacity references, freeing where none remain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) free_sched_groups(sd->groups, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) kfree(sd->shared);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) kfree(sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) static void destroy_sched_domains_rcu(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) while (sd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) struct sched_domain *parent = sd->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) destroy_sched_domain(sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) sd = parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) static void destroy_sched_domains(struct sched_domain *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) if (sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) call_rcu(&sd->rcu, destroy_sched_domains_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) * Keep a special pointer to the highest sched_domain that has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * allows us to avoid some pointer chasing select_idle_sibling().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * Also keep a unique ID per domain (we use the first CPU number in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * the cpumask of the domain), this allows us to quickly tell if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * two CPUs are in the same cache domain, see cpus_share_cache().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) DEFINE_PER_CPU(int, sd_llc_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) DEFINE_PER_CPU(int, sd_llc_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) static void update_top_cache_domain(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) struct sched_domain_shared *sds = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) struct sched_domain *sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) int id = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) int size = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) if (sd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) id = cpumask_first(sched_domain_span(sd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) size = cpumask_weight(sched_domain_span(sd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) sds = sd->shared;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) per_cpu(sd_llc_size, cpu) = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) per_cpu(sd_llc_id, cpu) = id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) sd = lowest_flag_domain(cpu, SD_NUMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * hold the hotplug lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) struct rq *rq = cpu_rq(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) struct sched_domain *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) /* Remove the sched domains which do not contribute to scheduling. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) for (tmp = sd; tmp; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) struct sched_domain *parent = tmp->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (!parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) if (sd_parent_degenerate(tmp, parent)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) tmp->parent = parent->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) if (parent->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) parent->parent->child = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) * Transfer SD_PREFER_SIBLING down in case of a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) * degenerate parent; the spans match for this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) * so the property transfers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) if (parent->flags & SD_PREFER_SIBLING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) tmp->flags |= SD_PREFER_SIBLING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) destroy_sched_domain(parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) tmp = tmp->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) if (sd && sd_degenerate(sd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) tmp = sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) sd = sd->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) destroy_sched_domain(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) if (sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) sd->child = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) sched_domain_debug(sd, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) rq_attach_root(rq, rd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) tmp = rq->sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) rcu_assign_pointer(rq->sd, sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) dirty_sched_domain_sysctl(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) destroy_sched_domains(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) update_top_cache_domain(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) struct s_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) struct sched_domain * __percpu *sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) struct root_domain *rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) enum s_alloc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) sa_rootdomain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) sa_sd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) sa_sd_storage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) sa_none,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) * Return the canonical balance CPU for this group, this is the first CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * of this group that's also in the balance mask.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) * The balance mask are all those CPUs that could actually end up at this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) * group. See build_balance_mask().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * Also see should_we_balance().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) int group_balance_cpu(struct sched_group *sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return cpumask_first(group_balance_mask(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) * NUMA topology (first read the regular topology blurb below)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * Given a node-distance table, for example:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * node 0 1 2 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * 0: 10 20 30 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * 1: 20 10 20 30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * 2: 30 20 10 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * 3: 20 30 20 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) * which represents a 4 node ring topology like:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * 0 ----- 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * 3 ----- 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * We want to construct domains and groups to represent this. The way we go
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * about doing this is to build the domains on 'hops'. For each NUMA level we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * construct the mask of all nodes reachable in @level hops.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * For the above NUMA topology that gives 3 levels:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * NUMA-2 0-3 0-3 0-3 0-3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * groups: {0-1,3},{1-3} {0-2},{0,2-3} {1-3},{0-1,3} {0,2-3},{0-2}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * NUMA-1 0-1,3 0-2 1-3 0,2-3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * groups: {0},{1},{3} {0},{1},{2} {1},{2},{3} {0},{2},{3}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * NUMA-0 0 1 2 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) * As can be seen; things don't nicely line up as with the regular topology.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) * When we iterate a domain in child domain chunks some nodes can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * represented multiple times -- hence the "overlap" naming for this part of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * the topology.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) * In order to minimize this overlap, we only build enough groups to cover the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * domain. For instance Node-0 NUMA-2 would only get groups: 0-1,3 and 1-3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * Because:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * - the first group of each domain is its child domain; this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * gets us the first 0-1,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) * - the only uncovered node is 2, who's child domain is 1-3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) * However, because of the overlap, computing a unique CPU for each group is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * more complicated. Consider for instance the groups of NODE-1 NUMA-2, both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * groups include the CPUs of Node-0, while those CPUs would not in fact ever
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * end up at those groups (they would end up in group: 0-1,3).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) * To correct this we have to introduce the group balance mask. This mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * will contain those CPUs in the group that can reach this group given the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) * (child) domain tree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * With this we can once again compute balance_cpu and sched_group_capacity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) * relations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * XXX include words on how balance_cpu is unique and therefore can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * used for sched_group_capacity links.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * Another 'interesting' topology is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * node 0 1 2 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) * 0: 10 20 20 30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) * 1: 20 10 20 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) * 2: 20 20 10 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) * 3: 30 20 20 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * Which looks a little like:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * 0 ----- 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * | / |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * | / |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * | / |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * 2 ----- 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) * This topology is asymmetric, nodes 1,2 are fully connected, but nodes 0,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) * are not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) * This leads to a few particularly weird cases where the sched_domain's are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) * not of the same number for each CPU. Consider:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) * NUMA-2 0-3 0-3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) * groups: {0-2},{1-3} {1-3},{0-2}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) * NUMA-1 0-2 0-3 0-3 1-3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) * NUMA-0 0 1 2 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * Build the balance mask; it contains only those CPUs that can arrive at this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * group and should be considered to continue balancing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * We do this during the group creation pass, therefore the group information
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * isn't complete yet, however since each group represents a (child) domain we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) * can fully construct this using the sched_domain bits (which are already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * complete).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) build_balance_mask(struct sched_domain *sd, struct sched_group *sg, struct cpumask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) const struct cpumask *sg_span = sched_group_span(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) struct sd_data *sdd = sd->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) struct sched_domain *sibling;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) cpumask_clear(mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) for_each_cpu(i, sg_span) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) sibling = *per_cpu_ptr(sdd->sd, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) * Can happen in the asymmetric case, where these siblings are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) * unused. The mask will not be empty because those CPUs that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * do have the top domain _should_ span the domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) if (!sibling->child)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) /* If we would not end up here, we can't continue from here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (!cpumask_equal(sg_span, sched_domain_span(sibling->child)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) cpumask_set_cpu(i, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) /* We must not have empty masks here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) WARN_ON_ONCE(cpumask_empty(mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * XXX: This creates per-node group entries; since the load-balancer will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * immediately access remote memory to construct this group's load-balance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * statistics having the groups node local is of dubious benefit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) static struct sched_group *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) build_group_from_child_sched_domain(struct sched_domain *sd, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) struct sched_group *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) struct cpumask *sg_span;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) GFP_KERNEL, cpu_to_node(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if (!sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) sg_span = sched_group_span(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) if (sd->child)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) cpumask_copy(sg_span, sched_domain_span(sd->child));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) cpumask_copy(sg_span, sched_domain_span(sd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) atomic_inc(&sg->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) return sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) static void init_overlap_sched_group(struct sched_domain *sd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) struct sched_group *sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) struct cpumask *mask = sched_domains_tmpmask2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) struct sd_data *sdd = sd->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) struct cpumask *sg_span;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) build_balance_mask(sd, sg, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) cpu = cpumask_first_and(sched_group_span(sg), mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) sg->sgc = *per_cpu_ptr(sdd->sgc, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) if (atomic_inc_return(&sg->sgc->ref) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) cpumask_copy(group_balance_mask(sg), mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) WARN_ON_ONCE(!cpumask_equal(group_balance_mask(sg), mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) * Initialize sgc->capacity such that even if we mess up the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) * domains and no possible iteration will get us here, we won't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) * die on a /0 trap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) sg_span = sched_group_span(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) sg->sgc->max_capacity = SCHED_CAPACITY_SCALE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) build_overlap_sched_groups(struct sched_domain *sd, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) struct sched_group *first = NULL, *last = NULL, *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) const struct cpumask *span = sched_domain_span(sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) struct cpumask *covered = sched_domains_tmpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) struct sd_data *sdd = sd->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) struct sched_domain *sibling;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) cpumask_clear(covered);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) for_each_cpu_wrap(i, span, cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) struct cpumask *sg_span;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) if (cpumask_test_cpu(i, covered))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) sibling = *per_cpu_ptr(sdd->sd, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * Asymmetric node setups can result in situations where the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * domain tree is of unequal depth, make sure to skip domains
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) * that already cover the entire range.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) * In that case build_sched_domains() will have terminated the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) * iteration early and our sibling sd spans will be empty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * Domains should always include the CPU they're built on, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * check that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) sg = build_group_from_child_sched_domain(sibling, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if (!sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) sg_span = sched_group_span(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) cpumask_or(covered, covered, sg_span);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) init_overlap_sched_group(sd, sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) if (!first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) first = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) if (last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) last->next = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) last = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) last->next = first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) sd->groups = first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) free_sched_groups(first, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) * Package topology (also see the load-balance blurb in fair.c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) * The scheduler builds a tree structure to represent a number of important
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) * topology features. By default (default_topology[]) these include:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) * - Simultaneous multithreading (SMT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) * - Multi-Core Cache (MC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) * - Package (DIE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) * Where the last one more or less denotes everything up to a NUMA node.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) * The tree consists of 3 primary data structures:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) * sched_domain -> sched_group -> sched_group_capacity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) * ^ ^ ^ ^
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) * `-' `-'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) * The sched_domains are per-CPU and have a two way link (parent & child) and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) * denote the ever growing mask of CPUs belonging to that level of topology.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) * Each sched_domain has a circular (double) linked list of sched_group's, each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) * denoting the domains of the level below (or individual CPUs in case of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * first domain level). The sched_group linked by a sched_domain includes the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) * CPU of that sched_domain [*].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) * Take for instance a 2 threaded, 2 core, 2 cache cluster part:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) * CPU 0 1 2 3 4 5 6 7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) * DIE [ ]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) * MC [ ] [ ]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) * SMT [ ] [ ] [ ] [ ]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) * - or -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) * DIE 0-7 0-7 0-7 0-7 0-7 0-7 0-7 0-7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) * MC 0-3 0-3 0-3 0-3 4-7 4-7 4-7 4-7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) * SMT 0-1 0-1 2-3 2-3 4-5 4-5 6-7 6-7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) * CPU 0 1 2 3 4 5 6 7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) * One way to think about it is: sched_domain moves you up and down among these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * topology levels, while sched_group moves you sideways through it, at child
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) * domain granularity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) * sched_group_capacity ensures each unique sched_group has shared storage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) * There are two related construction problems, both require a CPU that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * uniquely identify each group (for a given domain):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * - The first is the balance_cpu (see should_we_balance() and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) * load-balance blub in fair.c); for each group we only want 1 CPU to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) * continue balancing at a higher domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) * - The second is the sched_group_capacity; we want all identical groups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) * to share a single sched_group_capacity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) * Since these topologies are exclusive by construction. That is, its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) * impossible for an SMT thread to belong to multiple cores, and cores to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) * be part of multiple caches. There is a very clear and unique location
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) * for each CPU in the hierarchy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) * Therefore computing a unique CPU for each group is trivial (the iteration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) * mask is redundant and set all 1s; all CPUs in a group will end up at _that_
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) * group), we can simply pick the first CPU in each group.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) * [*] in other words, the first group of each domain is its child domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) static struct sched_group *get_group(int cpu, struct sd_data *sdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) struct sched_domain *child = sd->child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) struct sched_group *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) bool already_visited;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) if (child)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) cpu = cpumask_first(sched_domain_span(child));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) sg = *per_cpu_ptr(sdd->sg, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) sg->sgc = *per_cpu_ptr(sdd->sgc, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) /* Increase refcounts for claim_allocations: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) already_visited = atomic_inc_return(&sg->ref) > 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) /* sgc visits should follow a similar trend as sg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) WARN_ON(already_visited != (atomic_inc_return(&sg->sgc->ref) > 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) /* If we have already visited that group, it's already initialized. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) if (already_visited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) return sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) if (child) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) cpumask_copy(sched_group_span(sg), sched_domain_span(child));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) cpumask_copy(group_balance_mask(sg), sched_group_span(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) cpumask_set_cpu(cpu, sched_group_span(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) cpumask_set_cpu(cpu, group_balance_mask(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sched_group_span(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) sg->sgc->max_capacity = SCHED_CAPACITY_SCALE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) return sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * build_sched_groups will build a circular linked list of the groups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) * covered by the given span, will set each group's ->cpumask correctly,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) * and will initialize their ->sgc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) * Assumes the sched_domain tree is fully constructed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) build_sched_groups(struct sched_domain *sd, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) struct sched_group *first = NULL, *last = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) struct sd_data *sdd = sd->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) const struct cpumask *span = sched_domain_span(sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) struct cpumask *covered;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) lockdep_assert_held(&sched_domains_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) covered = sched_domains_tmpmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) cpumask_clear(covered);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) for_each_cpu_wrap(i, span, cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) struct sched_group *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) if (cpumask_test_cpu(i, covered))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) sg = get_group(i, sdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) cpumask_or(covered, covered, sched_group_span(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) if (!first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) first = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) if (last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) last->next = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) last = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) last->next = first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) sd->groups = first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) * Initialize sched groups cpu_capacity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) * cpu_capacity indicates the capacity of sched group, which is used while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) * distributing the load between different sched groups in a sched domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) * Typically cpu_capacity for all the groups in a sched domain will be same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) * unless there are asymmetries in the topology. If there are asymmetries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) * group having more cpu_capacity will pickup more load compared to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) * group having less cpu_capacity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) struct sched_group *sg = sd->groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) WARN_ON(!sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) int cpu, max_cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) sg->group_weight = cpumask_weight(sched_group_span(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) if (!(sd->flags & SD_ASYM_PACKING))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) for_each_cpu(cpu, sched_group_span(sg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (max_cpu < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) max_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) else if (sched_asym_prefer(cpu, max_cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) max_cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) sg->asym_prefer_cpu = max_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) sg = sg->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) } while (sg != sd->groups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) if (cpu != group_balance_cpu(sg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) update_group_capacity(sd, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) * Initializers for schedule domains
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) static int default_relax_domain_level = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) int sched_domain_level_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) static int __init setup_relax_domain_level(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (kstrtoint(str, 0, &default_relax_domain_level))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) pr_warn("Unable to set relax_domain_level\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) __setup("relax_domain_level=", setup_relax_domain_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) static void set_domain_attribute(struct sched_domain *sd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) struct sched_domain_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) int request;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) if (!attr || attr->relax_domain_level < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) if (default_relax_domain_level < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) request = default_relax_domain_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) request = attr->relax_domain_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (sd->level > request) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) /* Turn off idle balance on this domain: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) static void __sdt_free(const struct cpumask *cpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) static int __sdt_alloc(const struct cpumask *cpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) const struct cpumask *cpu_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) switch (what) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) case sa_rootdomain:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) if (!atomic_read(&d->rd->refcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) free_rootdomain(&d->rd->rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) case sa_sd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) free_percpu(d->sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) case sa_sd_storage:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) __sdt_free(cpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) case sa_none:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) static enum s_alloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) __visit_domain_allocation_hell(struct s_data *d, const struct cpumask *cpu_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) memset(d, 0, sizeof(*d));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) if (__sdt_alloc(cpu_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) return sa_sd_storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) d->sd = alloc_percpu(struct sched_domain *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) if (!d->sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) return sa_sd_storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) d->rd = alloc_rootdomain();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) if (!d->rd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) return sa_sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) return sa_rootdomain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) * NULL the sd_data elements we've used to build the sched_domain and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) * sched_group structure so that the subsequent __free_domain_allocs()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) * will not free the data we're using.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) static void claim_allocations(int cpu, struct sched_domain *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) struct sd_data *sdd = sd->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) *per_cpu_ptr(sdd->sd, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) *per_cpu_ptr(sdd->sds, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) *per_cpu_ptr(sdd->sg, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) *per_cpu_ptr(sdd->sgc, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) enum numa_topology_type sched_numa_topology_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) static int sched_domains_numa_levels;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) static int sched_domains_curr_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) int sched_max_numa_distance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) static int *sched_domains_numa_distance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) static struct cpumask ***sched_domains_numa_masks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) * SD_flags allowed in topology descriptions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) * These flags are purely descriptive of the topology and do not prescribe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * behaviour. Behaviour is artificial and mapped in the below sd_init()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) * function:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) * SD_SHARE_CPUCAPACITY - describes SMT topologies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) * SD_SHARE_PKG_RESOURCES - describes shared caches
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) * SD_NUMA - describes NUMA topologies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) * Odd one out, which beside describing the topology has a quirk also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) * prescribes the desired behaviour that goes along with it:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) * SD_ASYM_PACKING - describes SMT quirks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) #define TOPOLOGY_SD_FLAGS \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) (SD_SHARE_CPUCAPACITY | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) SD_SHARE_PKG_RESOURCES | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) SD_NUMA | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) SD_ASYM_PACKING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) static struct sched_domain *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) sd_init(struct sched_domain_topology_level *tl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) const struct cpumask *cpu_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) struct sched_domain *child, int dflags, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) struct sd_data *sdd = &tl->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) int sd_id, sd_weight, sd_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) * Ugly hack to pass state to sd_numa_mask()...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) sched_domains_curr_level = tl->numa_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) sd_weight = cpumask_weight(tl->mask(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) if (tl->sd_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) sd_flags = (*tl->sd_flags)();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) "wrong sd_flags in topology description\n"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) sd_flags &= TOPOLOGY_SD_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) /* Apply detected topology flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) sd_flags |= dflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) *sd = (struct sched_domain){
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) .min_interval = sd_weight,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) .max_interval = 2*sd_weight,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) .busy_factor = 16,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) .imbalance_pct = 117,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) .cache_nice_tries = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) .flags = 1*SD_BALANCE_NEWIDLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) | 1*SD_BALANCE_EXEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) | 1*SD_BALANCE_FORK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) | 0*SD_BALANCE_WAKE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) | 1*SD_WAKE_AFFINE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) | 0*SD_SHARE_CPUCAPACITY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) | 0*SD_SHARE_PKG_RESOURCES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) | 0*SD_SERIALIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) | 1*SD_PREFER_SIBLING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) | 0*SD_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) | sd_flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) .last_balance = jiffies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) .balance_interval = sd_weight,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) .max_newidle_lb_cost = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) .next_decay_max_lb_cost = jiffies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) .child = child,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) #ifdef CONFIG_SCHED_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) .name = tl->name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) sd_id = cpumask_first(sched_domain_span(sd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) * Convert topological properties into behaviour.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) /* Don't attempt to spread across CPUs of different capacities. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) if ((sd->flags & SD_ASYM_CPUCAPACITY) && sd->child)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) sd->child->flags &= ~SD_PREFER_SIBLING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) if (sd->flags & SD_SHARE_CPUCAPACITY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) sd->imbalance_pct = 110;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) } else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) sd->imbalance_pct = 117;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) sd->cache_nice_tries = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) } else if (sd->flags & SD_NUMA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) sd->cache_nice_tries = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) sd->flags &= ~SD_PREFER_SIBLING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) sd->flags |= SD_SERIALIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) if (sched_domains_numa_distance[tl->numa_level] > node_reclaim_distance) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) sd->flags &= ~(SD_BALANCE_EXEC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) SD_BALANCE_FORK |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) SD_WAKE_AFFINE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) sd->cache_nice_tries = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) * For all levels sharing cache; connect a sched_domain_shared
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) * instance.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) if (sd->flags & SD_SHARE_PKG_RESOURCES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) sd->shared = *per_cpu_ptr(sdd->sds, sd_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) atomic_inc(&sd->shared->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) atomic_set(&sd->shared->nr_busy_cpus, sd_weight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) sd->private = sdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) return sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) * Topology list, bottom-up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) static struct sched_domain_topology_level default_topology[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) #ifdef CONFIG_SCHED_SMT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) #ifdef CONFIG_SCHED_MC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) { cpu_cpu_mask, SD_INIT_NAME(DIE) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) { NULL, },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) static struct sched_domain_topology_level *sched_domain_topology =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) default_topology;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) #define for_each_sd_topology(tl) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) for (tl = sched_domain_topology; tl->mask; tl++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) void set_sched_topology(struct sched_domain_topology_level *tl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) if (WARN_ON_ONCE(sched_smp_initialized))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) sched_domain_topology = tl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) static const struct cpumask *sd_numa_mask(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) static void sched_numa_warn(const char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) static int done = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) int i,j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) if (done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) printk(KERN_WARNING "ERROR: %s\n\n", str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) for (i = 0; i < nr_node_ids; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) printk(KERN_WARNING " ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) for (j = 0; j < nr_node_ids; j++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) printk(KERN_CONT "%02d ", node_distance(i,j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) printk(KERN_CONT "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) printk(KERN_WARNING "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) bool find_numa_distance(int distance)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) if (distance == node_distance(0, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) for (i = 0; i < sched_domains_numa_levels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) if (sched_domains_numa_distance[i] == distance)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) * A system can have three types of NUMA topology:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) * NUMA_BACKPLANE: nodes can reach other nodes through a backplane
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) * The difference between a glueless mesh topology and a backplane
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) * topology lies in whether communication between not directly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) * connected nodes goes through intermediary nodes (where programs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) * could run), or through backplane controllers. This affects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) * placement of programs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) * The type of topology can be discerned with the following tests:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) * - If the maximum distance between any nodes is 1 hop, the system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) * is directly connected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) * - If for two nodes A and B, located N > 1 hops away from each other,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) * there is an intermediary node C, which is < N hops away from both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) * nodes A and B, the system is a glueless mesh.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) static void init_numa_topology_type(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) int a, b, c, n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) n = sched_max_numa_distance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) if (sched_domains_numa_levels <= 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) sched_numa_topology_type = NUMA_DIRECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) for_each_online_node(a) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) for_each_online_node(b) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) /* Find two nodes furthest removed from each other. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) if (node_distance(a, b) < n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) /* Is there an intermediary node between a and b? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) for_each_online_node(c) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) if (node_distance(a, c) < n &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) node_distance(b, c) < n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) sched_numa_topology_type =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) NUMA_GLUELESS_MESH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) sched_numa_topology_type = NUMA_BACKPLANE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) #define NR_DISTANCE_VALUES (1 << DISTANCE_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) void sched_init_numa(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) struct sched_domain_topology_level *tl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) unsigned long *distance_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) int nr_levels = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) * O(nr_nodes^2) deduplicating selection sort -- in order to find the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) * unique distances in the node_distance() table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) distance_map = bitmap_alloc(NR_DISTANCE_VALUES, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) if (!distance_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) bitmap_zero(distance_map, NR_DISTANCE_VALUES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) for (i = 0; i < nr_node_ids; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) for (j = 0; j < nr_node_ids; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) int distance = node_distance(i, j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) if (distance < LOCAL_DISTANCE || distance >= NR_DISTANCE_VALUES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) sched_numa_warn("Invalid distance value range");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) bitmap_set(distance_map, distance, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) * We can now figure out how many unique distance values there are and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) * allocate memory accordingly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) nr_levels = bitmap_weight(distance_map, NR_DISTANCE_VALUES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) sched_domains_numa_distance = kcalloc(nr_levels, sizeof(int), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) if (!sched_domains_numa_distance) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) bitmap_free(distance_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) for (i = 0, j = 0; i < nr_levels; i++, j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) j = find_next_bit(distance_map, NR_DISTANCE_VALUES, j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) sched_domains_numa_distance[i] = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) bitmap_free(distance_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) * 'nr_levels' contains the number of unique distances
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) * The sched_domains_numa_distance[] array includes the actual distance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) * numbers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) * Here, we should temporarily reset sched_domains_numa_levels to 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) * If it fails to allocate memory for array sched_domains_numa_masks[][],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) * the array will contain less then 'nr_levels' members. This could be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) * dangerous when we use it to iterate array sched_domains_numa_masks[][]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) * in other functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) * We reset it to 'nr_levels' at the end of this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) sched_domains_numa_levels = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) sched_domains_numa_masks = kzalloc(sizeof(void *) * nr_levels, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) if (!sched_domains_numa_masks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) * Now for each level, construct a mask per node which contains all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) * CPUs of nodes that are that many hops away from us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) for (i = 0; i < nr_levels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) sched_domains_numa_masks[i] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) if (!sched_domains_numa_masks[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) for (j = 0; j < nr_node_ids; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) int k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) if (!mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) sched_domains_numa_masks[i][j] = mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) for_each_node(k) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) if (sched_debug() && (node_distance(j, k) != node_distance(k, j)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) sched_numa_warn("Node-distance not symmetric");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) if (node_distance(j, k) > sched_domains_numa_distance[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) cpumask_or(mask, mask, cpumask_of_node(k));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) /* Compute default topology size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) for (i = 0; sched_domain_topology[i].mask; i++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) tl = kzalloc((i + nr_levels + 1) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) sizeof(struct sched_domain_topology_level), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) if (!tl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) * Copy the default topology bits..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) for (i = 0; sched_domain_topology[i].mask; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) tl[i] = sched_domain_topology[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) * Add the NUMA identity distance, aka single NODE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) tl[i++] = (struct sched_domain_topology_level){
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) .mask = sd_numa_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) .numa_level = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) SD_INIT_NAME(NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) * .. and append 'j' levels of NUMA goodness.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) for (j = 1; j < nr_levels; i++, j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) tl[i] = (struct sched_domain_topology_level){
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) .mask = sd_numa_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) .sd_flags = cpu_numa_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) .flags = SDTL_OVERLAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) .numa_level = j,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) SD_INIT_NAME(NUMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) sched_domain_topology = tl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) sched_domains_numa_levels = nr_levels;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) sched_max_numa_distance = sched_domains_numa_distance[nr_levels - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) init_numa_topology_type();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) void sched_domains_numa_masks_set(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) int node = cpu_to_node(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) for (i = 0; i < sched_domains_numa_levels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) for (j = 0; j < nr_node_ids; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) if (node_distance(j, node) <= sched_domains_numa_distance[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) void sched_domains_numa_masks_clear(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) for (i = 0; i < sched_domains_numa_levels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) for (j = 0; j < nr_node_ids; j++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) * sched_numa_find_closest() - given the NUMA topology, find the cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) * closest to @cpu from @cpumask.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) * cpumask: cpumask to find a cpu from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) * cpu: cpu to be close to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) * returns: cpu, or nr_cpu_ids when nothing found.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) int i, j = cpu_to_node(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) for (i = 0; i < sched_domains_numa_levels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) cpu = cpumask_any_and(cpus, sched_domains_numa_masks[i][j]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) if (cpu < nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) return cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) return nr_cpu_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) #endif /* CONFIG_NUMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) static int __sdt_alloc(const struct cpumask *cpu_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) struct sched_domain_topology_level *tl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) int j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) for_each_sd_topology(tl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) struct sd_data *sdd = &tl->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) sdd->sd = alloc_percpu(struct sched_domain *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) if (!sdd->sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) sdd->sds = alloc_percpu(struct sched_domain_shared *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) if (!sdd->sds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) sdd->sg = alloc_percpu(struct sched_group *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) if (!sdd->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) sdd->sgc = alloc_percpu(struct sched_group_capacity *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) if (!sdd->sgc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) for_each_cpu(j, cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) struct sched_domain *sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) struct sched_domain_shared *sds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) struct sched_group *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) struct sched_group_capacity *sgc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) GFP_KERNEL, cpu_to_node(j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) if (!sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) *per_cpu_ptr(sdd->sd, j) = sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) sds = kzalloc_node(sizeof(struct sched_domain_shared),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) GFP_KERNEL, cpu_to_node(j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) if (!sds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) *per_cpu_ptr(sdd->sds, j) = sds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) GFP_KERNEL, cpu_to_node(j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) if (!sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) sg->next = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) *per_cpu_ptr(sdd->sg, j) = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) GFP_KERNEL, cpu_to_node(j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) if (!sgc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) #ifdef CONFIG_SCHED_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) sgc->id = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) *per_cpu_ptr(sdd->sgc, j) = sgc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) static void __sdt_free(const struct cpumask *cpu_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) struct sched_domain_topology_level *tl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) int j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) for_each_sd_topology(tl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) struct sd_data *sdd = &tl->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) for_each_cpu(j, cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) struct sched_domain *sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) if (sdd->sd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) sd = *per_cpu_ptr(sdd->sd, j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) if (sd && (sd->flags & SD_OVERLAP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) free_sched_groups(sd->groups, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) kfree(*per_cpu_ptr(sdd->sd, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) if (sdd->sds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) kfree(*per_cpu_ptr(sdd->sds, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) if (sdd->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) kfree(*per_cpu_ptr(sdd->sg, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) if (sdd->sgc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) kfree(*per_cpu_ptr(sdd->sgc, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) free_percpu(sdd->sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) sdd->sd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) free_percpu(sdd->sds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) sdd->sds = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) free_percpu(sdd->sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) sdd->sg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) free_percpu(sdd->sgc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) sdd->sgc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) static struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) const struct cpumask *cpu_map, struct sched_domain_attr *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) struct sched_domain *child, int dflags, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) struct sched_domain *sd = sd_init(tl, cpu_map, child, dflags, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) if (child) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) sd->level = child->level + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) sched_domain_level_max = max(sched_domain_level_max, sd->level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) child->parent = sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) if (!cpumask_subset(sched_domain_span(child),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) sched_domain_span(sd))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) pr_err("BUG: arch topology borken\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) #ifdef CONFIG_SCHED_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) pr_err(" the %s domain not a subset of the %s domain\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) child->name, sd->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) /* Fixup, ensure @sd has at least @child CPUs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) cpumask_or(sched_domain_span(sd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) sched_domain_span(sd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) sched_domain_span(child));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) set_domain_attribute(sd, attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) return sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) * Ensure topology masks are sane, i.e. there are no conflicts (overlaps) for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) * any two given CPUs at this (non-NUMA) topology level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) static bool topology_span_sane(struct sched_domain_topology_level *tl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) const struct cpumask *cpu_map, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) /* NUMA levels are allowed to overlap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) if (tl->flags & SDTL_OVERLAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) * Non-NUMA levels cannot partially overlap - they must be either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) * completely equal or completely disjoint. Otherwise we can end up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) * breaking the sched_group lists - i.e. a later get_group() pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) * breaks the linking done for an earlier span.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) for_each_cpu(i, cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) if (i == cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) * We should 'and' all those masks with 'cpu_map' to exactly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) * match the topology we're about to build, but that can only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) * remove CPUs, which only lessens our ability to detect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) * overlaps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) if (!cpumask_equal(tl->mask(cpu), tl->mask(i)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) cpumask_intersects(tl->mask(cpu), tl->mask(i)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) * Find the sched_domain_topology_level where all CPU capacities are visible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) * for all CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) static struct sched_domain_topology_level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) *asym_cpu_capacity_level(const struct cpumask *cpu_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) int i, j, asym_level = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) bool asym = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) struct sched_domain_topology_level *tl, *asym_tl = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) unsigned long cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) /* Is there any asymmetry? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) cap = arch_scale_cpu_capacity(cpumask_first(cpu_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) for_each_cpu(i, cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) if (arch_scale_cpu_capacity(i) != cap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) asym = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) if (!asym)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) * Examine topology from all CPU's point of views to detect the lowest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) * sched_domain_topology_level where a highest capacity CPU is visible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) * to everyone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) for_each_cpu(i, cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) unsigned long max_capacity = arch_scale_cpu_capacity(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) int tl_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) for_each_sd_topology(tl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) if (tl_id < asym_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) goto next_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) for_each_cpu_and(j, tl->mask(i), cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) unsigned long capacity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) capacity = arch_scale_cpu_capacity(j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) if (capacity <= max_capacity)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) max_capacity = capacity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) asym_level = tl_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) asym_tl = tl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) next_level:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) tl_id++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) return asym_tl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) * Build sched domains for a given set of CPUs and attach the sched domains
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) * to the individual CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) enum s_alloc alloc_state = sa_none;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) struct sched_domain *sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) struct s_data d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) struct rq *rq = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) int i, ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) struct sched_domain_topology_level *tl_asym;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) bool has_asym = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) if (WARN_ON(cpumask_empty(cpu_map)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) if (alloc_state != sa_rootdomain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) tl_asym = asym_cpu_capacity_level(cpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) /* Set up domains for CPUs specified by the cpu_map: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) for_each_cpu(i, cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) struct sched_domain_topology_level *tl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) int dflags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) sd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) for_each_sd_topology(tl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) if (tl == tl_asym) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) dflags |= SD_ASYM_CPUCAPACITY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) has_asym = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) if (WARN_ON(!topology_span_sane(tl, cpu_map, i)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) sd = build_sched_domain(tl, cpu_map, attr, sd, dflags, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) if (tl == sched_domain_topology)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) *per_cpu_ptr(d.sd, i) = sd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) if (tl->flags & SDTL_OVERLAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) sd->flags |= SD_OVERLAP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) if (cpumask_equal(cpu_map, sched_domain_span(sd)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) /* Build the groups for the domains */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) for_each_cpu(i, cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) sd->span_weight = cpumask_weight(sched_domain_span(sd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) if (sd->flags & SD_OVERLAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) if (build_overlap_sched_groups(sd, i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) if (build_sched_groups(sd, i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) /* Calculate CPU capacity for physical packages and nodes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) for (i = nr_cpumask_bits-1; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) if (!cpumask_test_cpu(i, cpu_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) claim_allocations(i, sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) init_sched_groups_capacity(i, sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) /* Attach the domains */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) for_each_cpu(i, cpu_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) rq = cpu_rq(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) sd = *per_cpu_ptr(d.sd, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) /* Use READ_ONCE()/WRITE_ONCE() to avoid load/store tearing: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) cpu_attach_domain(sd, d.rd, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) if (has_asym)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) static_branch_inc_cpuslocked(&sched_asym_cpucapacity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) if (rq && sched_debug_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) trace_android_vh_build_sched_domains(has_asym);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) __free_domain_allocs(&d, alloc_state, cpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) /* Current sched domains: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) static cpumask_var_t *doms_cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) /* Number of sched domains in 'doms_cur': */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) static int ndoms_cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) /* Attribues of custom domains in 'doms_cur' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) static struct sched_domain_attr *dattr_cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) * Special case: If a kmalloc() of a doms_cur partition (array of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) * cpumask) fails, then fallback to a single sched domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) * as determined by the single cpumask fallback_doms.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) static cpumask_var_t fallback_doms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) * arch_update_cpu_topology lets virtualized architectures update the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) * CPU core maps. It is supposed to return 1 if the topology changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) * or 0 if it stayed the same.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) int __weak arch_update_cpu_topology(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) cpumask_var_t *doms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) doms = kmalloc_array(ndoms, sizeof(*doms), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) if (!doms)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) for (i = 0; i < ndoms; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) free_sched_domains(doms, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) return doms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) for (i = 0; i < ndoms; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) free_cpumask_var(doms[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) kfree(doms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) * Set up scheduler domains and groups. For now this just excludes isolated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) * CPUs, but could be used to exclude other special cases in the future.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) int sched_init_domains(const struct cpumask *cpu_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) zalloc_cpumask_var(&sched_domains_tmpmask, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) zalloc_cpumask_var(&sched_domains_tmpmask2, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) zalloc_cpumask_var(&fallback_doms, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) arch_update_cpu_topology();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) ndoms_cur = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) doms_cur = alloc_sched_domains(ndoms_cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) if (!doms_cur)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) doms_cur = &fallback_doms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) cpumask_and(doms_cur[0], cpu_map, housekeeping_cpumask(HK_FLAG_DOMAIN));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) err = build_sched_domains(doms_cur[0], NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) register_sched_domain_sysctl();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) * Detach sched domains from a group of CPUs specified in cpu_map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) * These CPUs will now be attached to the NULL domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) static void detach_destroy_domains(const struct cpumask *cpu_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) unsigned int cpu = cpumask_any(cpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) if (rcu_access_pointer(per_cpu(sd_asym_cpucapacity, cpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) static_branch_dec_cpuslocked(&sched_asym_cpucapacity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) for_each_cpu(i, cpu_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) cpu_attach_domain(NULL, &def_root_domain, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) /* handle null as "default" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) struct sched_domain_attr *new, int idx_new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) struct sched_domain_attr tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) /* Fast path: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) if (!new && !cur)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) tmp = SD_ATTR_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) return !memcmp(cur ? (cur + idx_cur) : &tmp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) new ? (new + idx_new) : &tmp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) sizeof(struct sched_domain_attr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) * Partition sched domains as specified by the 'ndoms_new'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) * cpumasks in the array doms_new[] of cpumasks. This compares
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) * doms_new[] to the current sched domain partitioning, doms_cur[].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) * It destroys each deleted domain and builds each new domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) * The masks don't intersect (don't overlap.) We should setup one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) * sched domain for each mask. CPUs not in any of the cpumasks will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) * not be load balanced. If the same cpumask appears both in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) * current 'doms_cur' domains and in the new 'doms_new', we can leave
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) * it as it is.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) * The passed in 'doms_new' should be allocated using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) * alloc_sched_domains. This routine takes ownership of it and will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) * free_sched_domains it when done with it. If the caller failed the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) * and partition_sched_domains() will fallback to the single partition
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) * 'fallback_doms', it also forces the domains to be rebuilt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) * If doms_new == NULL it will be replaced with cpu_online_mask.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) * ndoms_new == 0 is a special case for destroying existing domains,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) * and it will not create the default domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) * Call with hotplug lock and sched_domains_mutex held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) struct sched_domain_attr *dattr_new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) bool __maybe_unused has_eas = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) int i, j, n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) int new_topology;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) lockdep_assert_held(&sched_domains_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) /* Always unregister in case we don't destroy any domains: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) unregister_sched_domain_sysctl();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) /* Let the architecture update CPU core mappings: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) new_topology = arch_update_cpu_topology();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) if (!doms_new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) WARN_ON_ONCE(dattr_new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) doms_new = alloc_sched_domains(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) if (doms_new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) n = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) cpumask_and(doms_new[0], cpu_active_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) housekeeping_cpumask(HK_FLAG_DOMAIN));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) n = ndoms_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) /* Destroy deleted domains: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) for (i = 0; i < ndoms_cur; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) for (j = 0; j < n && !new_topology; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) if (cpumask_equal(doms_cur[i], doms_new[j]) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) dattrs_equal(dattr_cur, i, dattr_new, j)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) struct root_domain *rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) * This domain won't be destroyed and as such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) * its dl_bw->total_bw needs to be cleared. It
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) * will be recomputed in function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) * update_tasks_root_domain().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) rd = cpu_rq(cpumask_any(doms_cur[i]))->rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) dl_clear_root_domain(rd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) goto match1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) /* No match - a current sched domain not in new doms_new[] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) detach_destroy_domains(doms_cur[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) match1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) n = ndoms_cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) if (!doms_new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) doms_new = &fallback_doms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) cpumask_and(doms_new[0], cpu_active_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) housekeeping_cpumask(HK_FLAG_DOMAIN));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) /* Build new domains: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) for (i = 0; i < ndoms_new; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) for (j = 0; j < n && !new_topology; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) if (cpumask_equal(doms_new[i], doms_cur[j]) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) dattrs_equal(dattr_new, i, dattr_cur, j))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) goto match2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) /* No match - add a new doms_new */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) match2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) /* Build perf. domains: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) for (i = 0; i < ndoms_new; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) for (j = 0; j < n && !sched_energy_update; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) if (cpumask_equal(doms_new[i], doms_cur[j]) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) cpu_rq(cpumask_first(doms_cur[j]))->rd->pd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) has_eas = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) goto match3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) /* No match - add perf. domains for a new rd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) has_eas |= build_perf_domains(doms_new[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) match3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) sched_energy_set(has_eas);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) /* Remember the new sched domains: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) if (doms_cur != &fallback_doms)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) free_sched_domains(doms_cur, ndoms_cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) kfree(dattr_cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) doms_cur = doms_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) dattr_cur = dattr_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) ndoms_cur = ndoms_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) register_sched_domain_sysctl();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) * Call with hotplug lock held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) struct sched_domain_attr *dattr_new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) mutex_lock(&sched_domains_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) mutex_unlock(&sched_domains_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags