^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Fast batching percpu counters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/percpu_counter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/debugobjects.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) static LIST_HEAD(percpu_counters);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) static DEFINE_SPINLOCK(percpu_counters_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) static const struct debug_obj_descr percpu_counter_debug_descr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) struct percpu_counter *fbc = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) switch (state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) percpu_counter_destroy(fbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) debug_object_free(fbc, &percpu_counter_debug_descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static const struct debug_obj_descr percpu_counter_debug_descr = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) .name = "percpu_counter",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) .fixup_free = percpu_counter_fixup_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) debug_object_init(fbc, &percpu_counter_debug_descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) debug_object_activate(fbc, &percpu_counter_debug_descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) debug_object_deactivate(fbc, &percpu_counter_debug_descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) debug_object_free(fbc, &percpu_counter_debug_descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #else /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #endif /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) raw_spin_lock_irqsave(&fbc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) *pcount = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) fbc->count = amount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) raw_spin_unlock_irqrestore(&fbc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) EXPORT_SYMBOL(percpu_counter_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * This function is both preempt and irq safe. The former is due to explicit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * preemption disable. The latter is guaranteed by the fact that the slow path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * is explicitly protected by an irq-safe spinlock whereas the fast patch uses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * this_cpu_add which is irq-safe by definition. Hence there is no need muck
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * with irq state before calling this one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) s64 count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) count = __this_cpu_read(*fbc->counters) + amount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (abs(count) >= batch) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) raw_spin_lock_irqsave(&fbc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) fbc->count += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) __this_cpu_sub(*fbc->counters, count - amount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) raw_spin_unlock_irqrestore(&fbc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) this_cpu_add(*fbc->counters, amount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) EXPORT_SYMBOL(percpu_counter_add_batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * For percpu_counter with a big batch, the devication of its count could
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * be big, and there is requirement to reduce the deviation, like when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * counter's batch could be runtime decreased to get a better accuracy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * which can be achieved by running this sync function on each CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) void percpu_counter_sync(struct percpu_counter *fbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) s64 count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) raw_spin_lock_irqsave(&fbc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) count = __this_cpu_read(*fbc->counters);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) fbc->count += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) __this_cpu_sub(*fbc->counters, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) raw_spin_unlock_irqrestore(&fbc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) EXPORT_SYMBOL(percpu_counter_sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * Add up all the per-cpu counts, return the result. This is a more accurate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * but much slower version of percpu_counter_read_positive()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) s64 __percpu_counter_sum(struct percpu_counter *fbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) s64 ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) raw_spin_lock_irqsave(&fbc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ret = fbc->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) ret += *pcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) raw_spin_unlock_irqrestore(&fbc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) EXPORT_SYMBOL(__percpu_counter_sum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) struct lock_class_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) unsigned long flags __maybe_unused;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) raw_spin_lock_init(&fbc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) lockdep_set_class(&fbc->lock, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) fbc->count = amount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) fbc->counters = alloc_percpu_gfp(s32, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (!fbc->counters)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) debug_percpu_counter_activate(fbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) INIT_LIST_HEAD(&fbc->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) spin_lock_irqsave(&percpu_counters_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) list_add(&fbc->list, &percpu_counters);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) spin_unlock_irqrestore(&percpu_counters_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) EXPORT_SYMBOL(__percpu_counter_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) void percpu_counter_destroy(struct percpu_counter *fbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) unsigned long flags __maybe_unused;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (!fbc->counters)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) debug_percpu_counter_deactivate(fbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) spin_lock_irqsave(&percpu_counters_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) list_del(&fbc->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) spin_unlock_irqrestore(&percpu_counters_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) free_percpu(fbc->counters);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) fbc->counters = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) EXPORT_SYMBOL(percpu_counter_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) int percpu_counter_batch __read_mostly = 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) EXPORT_SYMBOL(percpu_counter_batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) static int compute_batch_value(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) int nr = num_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) percpu_counter_batch = max(32, nr*2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) static int percpu_counter_cpu_dead(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct percpu_counter *fbc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) compute_batch_value(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) spin_lock_irq(&percpu_counters_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) list_for_each_entry(fbc, &percpu_counters, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) s32 *pcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) raw_spin_lock(&fbc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) pcount = per_cpu_ptr(fbc->counters, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) fbc->count += *pcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) *pcount = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) raw_spin_unlock(&fbc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) spin_unlock_irq(&percpu_counters_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * Compare counter against given value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * Return 1 if greater, 0 if equal and -1 if less
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) s64 count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) count = percpu_counter_read(fbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) /* Check to see if rough count will be sufficient for comparison */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) if (abs(count - rhs) > (batch * num_online_cpus())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (count > rhs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /* Need to use precise count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) count = percpu_counter_sum(fbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (count > rhs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) else if (count < rhs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) EXPORT_SYMBOL(__percpu_counter_compare);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static int __init percpu_counter_startup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "lib/percpu_cnt:online",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) compute_batch_value, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) WARN_ON(ret < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) ret = cpuhp_setup_state_nocalls(CPUHP_PERCPU_CNT_DEAD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) "lib/percpu_cnt:dead", NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) percpu_counter_cpu_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) WARN_ON(ret < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) module_init(percpu_counter_startup);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags