^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2016 Thomas Gleixner.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2016-2017 Christoph Hellwig.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/sort.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) unsigned int cpus_per_vec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) const struct cpumask *siblmsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) int cpu, sibl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) for ( ; cpus_per_vec > 0; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) cpu = cpumask_first(nmsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) /* Should not happen, but I'm too lazy to think about it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) if (cpu >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) cpumask_clear_cpu(cpu, nmsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) cpumask_set_cpu(cpu, irqmsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) cpus_per_vec--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /* If the cpu has siblings, use them first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) siblmsk = topology_sibling_cpumask(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) for (sibl = -1; cpus_per_vec > 0; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) sibl = cpumask_next(sibl, siblmsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) if (sibl >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) if (!cpumask_test_and_clear_cpu(sibl, nmsk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) cpumask_set_cpu(sibl, irqmsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) cpus_per_vec--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static cpumask_var_t *alloc_node_to_cpumask(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) cpumask_var_t *masks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) masks = kcalloc(nr_node_ids, sizeof(cpumask_var_t), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) if (!masks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) for (node = 0; node < nr_node_ids; node++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) if (!zalloc_cpumask_var(&masks[node], GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) goto out_unwind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) return masks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) out_unwind:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) while (--node >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) free_cpumask_var(masks[node]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) kfree(masks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static void free_node_to_cpumask(cpumask_var_t *masks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) for (node = 0; node < nr_node_ids; node++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) free_cpumask_var(masks[node]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) kfree(masks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static void build_node_to_cpumask(cpumask_var_t *masks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) cpumask_set_cpu(cpu, masks[cpu_to_node(cpu)]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) const struct cpumask *mask, nodemask_t *nodemsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) int n, nodes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /* Calculate the number of nodes in the supplied affinity mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) for_each_node(n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (cpumask_intersects(mask, node_to_cpumask[n])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) node_set(n, *nodemsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) nodes++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return nodes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct node_vectors {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) unsigned id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) unsigned nvectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) unsigned ncpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) static int ncpus_cmp_func(const void *l, const void *r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) const struct node_vectors *ln = l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) const struct node_vectors *rn = r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return ln->ncpus - rn->ncpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * Allocate vector number for each node, so that for each node:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * 1) the allocated number is >= 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * 2) the allocated numbver is <= active CPU number of this node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * The actual allocated total vectors may be less than @numvecs when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * active total CPU number is less than @numvecs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * Active CPUs means the CPUs in '@cpu_mask AND @node_to_cpumask[]'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * for each node.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) static void alloc_nodes_vectors(unsigned int numvecs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) cpumask_var_t *node_to_cpumask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) const struct cpumask *cpu_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) const nodemask_t nodemsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct cpumask *nmsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct node_vectors *node_vectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) unsigned n, remaining_ncpus = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) for (n = 0; n < nr_node_ids; n++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) node_vectors[n].id = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) node_vectors[n].ncpus = UINT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) for_each_node_mask(n, nodemsk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) unsigned ncpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) ncpus = cpumask_weight(nmsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (!ncpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) remaining_ncpus += ncpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) node_vectors[n].ncpus = ncpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) numvecs = min_t(unsigned, remaining_ncpus, numvecs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) sort(node_vectors, nr_node_ids, sizeof(node_vectors[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) ncpus_cmp_func, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * Allocate vectors for each node according to the ratio of this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * node's nr_cpus to remaining un-assigned ncpus. 'numvecs' is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * bigger than number of active numa nodes. Always start the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * allocation from the node with minimized nr_cpus.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * This way guarantees that each active node gets allocated at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * least one vector, and the theory is simple: over-allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * is only done when this node is assigned by one vector, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * other nodes will be allocated >= 1 vector, since 'numvecs' is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * bigger than number of numa nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * One perfect invariant is that number of allocated vectors for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * each node is <= CPU count of this node:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * 1) suppose there are two nodes: A and B
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * ncpu(X) is CPU count of node X
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * vecs(X) is the vector count allocated to node X via this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * algorithm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * ncpu(A) <= ncpu(B)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * ncpu(A) + ncpu(B) = N
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * vecs(A) + vecs(B) = V
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * vecs(A) = max(1, round_down(V * ncpu(A) / N))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * vecs(B) = V - vecs(A)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * both N and V are integer, and 2 <= V <= N, suppose
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * V = N - delta, and 0 <= delta <= N - 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * 2) obviously vecs(A) <= ncpu(A) because:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * if vecs(A) is 1, then vecs(A) <= ncpu(A) given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * ncpu(A) >= 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * otherwise,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * vecs(A) <= V * ncpu(A) / N <= ncpu(A), given V <= N
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * 3) prove how vecs(B) <= ncpu(B):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * if round_down(V * ncpu(A) / N) == 0, vecs(B) won't be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * over-allocated, so vecs(B) <= ncpu(B),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * otherwise:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * vecs(A) =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * round_down(V * ncpu(A) / N) =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * round_down((N - delta) * ncpu(A) / N) =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * round_down((N * ncpu(A) - delta * ncpu(A)) / N) >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * round_down((N * ncpu(A) - delta * N) / N) =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * cpu(A) - delta
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * then:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * vecs(A) - V >= ncpu(A) - delta - V
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * =>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * V - vecs(A) <= V + delta - ncpu(A)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * =>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * vecs(B) <= N - ncpu(A)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * =>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * vecs(B) <= cpu(B)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * For nodes >= 3, it can be thought as one node and another big
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * node given that is exactly what this algorithm is implemented,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * and we always re-calculate 'remaining_ncpus' & 'numvecs', and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * finally for each node X: vecs(X) <= ncpu(X).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) for (n = 0; n < nr_node_ids; n++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) unsigned nvectors, ncpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (node_vectors[n].ncpus == UINT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) WARN_ON_ONCE(numvecs == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) ncpus = node_vectors[n].ncpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) nvectors = max_t(unsigned, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) numvecs * ncpus / remaining_ncpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) WARN_ON_ONCE(nvectors > ncpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) node_vectors[n].nvectors = nvectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) remaining_ncpus -= ncpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) numvecs -= nvectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) static int __irq_build_affinity_masks(unsigned int startvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) unsigned int numvecs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) unsigned int firstvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) cpumask_var_t *node_to_cpumask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) const struct cpumask *cpu_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct cpumask *nmsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) struct irq_affinity_desc *masks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) unsigned int i, n, nodes, cpus_per_vec, extra_vecs, done = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) unsigned int last_affv = firstvec + numvecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) unsigned int curvec = startvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) nodemask_t nodemsk = NODE_MASK_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct node_vectors *node_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (!cpumask_weight(cpu_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) nodes = get_nodes_in_cpumask(node_to_cpumask, cpu_mask, &nodemsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * If the number of nodes in the mask is greater than or equal the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * number of vectors we just spread the vectors across the nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (numvecs <= nodes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) for_each_node_mask(n, nodemsk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) cpumask_or(&masks[curvec].mask, &masks[curvec].mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) node_to_cpumask[n]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (++curvec == last_affv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) curvec = firstvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return numvecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) node_vectors = kcalloc(nr_node_ids,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) sizeof(struct node_vectors),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) if (!node_vectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /* allocate vector number for each node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) alloc_nodes_vectors(numvecs, node_to_cpumask, cpu_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) nodemsk, nmsk, node_vectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) for (i = 0; i < nr_node_ids; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) unsigned int ncpus, v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) struct node_vectors *nv = &node_vectors[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) if (nv->nvectors == UINT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /* Get the cpus on this node which are in the mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) cpumask_and(nmsk, cpu_mask, node_to_cpumask[nv->id]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) ncpus = cpumask_weight(nmsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (!ncpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) WARN_ON_ONCE(nv->nvectors > ncpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) /* Account for rounding errors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) extra_vecs = ncpus - nv->nvectors * (ncpus / nv->nvectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* Spread allocated vectors on CPUs of the current node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) for (v = 0; v < nv->nvectors; v++, curvec++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) cpus_per_vec = ncpus / nv->nvectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /* Account for extra vectors to compensate rounding errors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (extra_vecs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) cpus_per_vec++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) --extra_vecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * wrapping has to be considered given 'startvec'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * may start anywhere
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (curvec >= last_affv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) curvec = firstvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) irq_spread_init_one(&masks[curvec].mask, nmsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) cpus_per_vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) done += nv->nvectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) kfree(node_vectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * build affinity in two stages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) * 1) spread present CPU on these vectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * 2) spread other possible CPUs on these vectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) static int irq_build_affinity_masks(unsigned int startvec, unsigned int numvecs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) unsigned int firstvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) struct irq_affinity_desc *masks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) unsigned int curvec = startvec, nr_present = 0, nr_others = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) cpumask_var_t *node_to_cpumask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) cpumask_var_t nmsk, npresmsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) goto fail_nmsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) node_to_cpumask = alloc_node_to_cpumask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (!node_to_cpumask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) goto fail_npresmsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) /* Stabilize the cpumasks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) get_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) build_node_to_cpumask(node_to_cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /* Spread on present CPUs starting from affd->pre_vectors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) ret = __irq_build_affinity_masks(curvec, numvecs, firstvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) node_to_cpumask, cpu_present_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) nmsk, masks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) goto fail_build_affinity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) nr_present = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * Spread on non present CPUs starting from the next vector to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * handled. If the spreading of present CPUs already exhausted the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * vector space, assign the non present CPUs to the already spread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * out vectors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (nr_present >= numvecs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) curvec = firstvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) curvec = firstvec + nr_present;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) ret = __irq_build_affinity_masks(curvec, numvecs, firstvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) node_to_cpumask, npresmsk, nmsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) masks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (ret >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) nr_others = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) fail_build_affinity:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) put_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) if (ret >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) WARN_ON(nr_present + nr_others < numvecs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) free_node_to_cpumask(node_to_cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) fail_npresmsk:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) free_cpumask_var(npresmsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) fail_nmsk:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) free_cpumask_var(nmsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) return ret < 0 ? ret : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) affd->nr_sets = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) affd->set_size[0] = affvecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * irq_create_affinity_masks - Create affinity masks for multiqueue spreading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * @nvecs: The total number of vectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * @affd: Description of the affinity requirements
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * Returns the irq_affinity_desc pointer or NULL if allocation failed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct irq_affinity_desc *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) irq_create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) unsigned int affvecs, curvec, usedvecs, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) struct irq_affinity_desc *masks = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * Determine the number of vectors which need interrupt affinities
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * assigned. If the pre/post request exhausts the available vectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * then nothing to do here except for invoking the calc_sets()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * callback so the device driver can adjust to the situation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (nvecs > affd->pre_vectors + affd->post_vectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) affvecs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * Simple invocations do not provide a calc_sets() callback. Install
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * the generic one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) if (!affd->calc_sets)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) affd->calc_sets = default_calc_sets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) /* Recalculate the sets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) affd->calc_sets(affd, affvecs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (WARN_ON_ONCE(affd->nr_sets > IRQ_AFFINITY_MAX_SETS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) /* Nothing to assign? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (!affvecs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (!masks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /* Fill out vectors at the beginning that don't need affinity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) for (curvec = 0; curvec < affd->pre_vectors; curvec++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) cpumask_copy(&masks[curvec].mask, irq_default_affinity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * Spread on present CPUs starting from affd->pre_vectors. If we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * have multiple sets, build each sets affinity mask separately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) unsigned int this_vecs = affd->set_size[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) ret = irq_build_affinity_masks(curvec, this_vecs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) curvec, masks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) kfree(masks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) curvec += this_vecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) usedvecs += this_vecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) /* Fill out vectors at the end that don't need affinity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (usedvecs >= affvecs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) curvec = affd->pre_vectors + affvecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) curvec = affd->pre_vectors + usedvecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) for (; curvec < nvecs; curvec++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) cpumask_copy(&masks[curvec].mask, irq_default_affinity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) /* Mark the managed interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) for (i = affd->pre_vectors; i < nvecs - affd->post_vectors; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) masks[i].is_managed = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) return masks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * irq_calc_affinity_vectors - Calculate the optimal number of vectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * @minvec: The minimum number of vectors available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * @maxvec: The maximum number of vectors available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * @affd: Description of the affinity requirements
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) unsigned int irq_calc_affinity_vectors(unsigned int minvec, unsigned int maxvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) const struct irq_affinity *affd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) unsigned int resv = affd->pre_vectors + affd->post_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) unsigned int set_vecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (resv > minvec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (affd->calc_sets) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) set_vecs = maxvec - resv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) get_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) set_vecs = cpumask_weight(cpu_possible_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) put_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) return resv + min(set_vecs, maxvec - resv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags