^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2016 Facebook
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2013-2014 Jens Axboe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/random.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/sbitmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * See if we have deferred clears that we can batch move
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) unsigned long mask, val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) spin_lock_irqsave(&sb->map[index].swap_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) if (!sb->map[index].cleared)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * First get a stable cleared mask, setting the old mask to 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) mask = xchg(&sb->map[index].cleared, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * Now clear the masked bits in our free word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) val = sb->map[index].word;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) } while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) spin_unlock_irqrestore(&sb->map[index].swap_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) gfp_t flags, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) unsigned int bits_per_word;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) if (shift < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) shift = ilog2(BITS_PER_LONG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * If the bitmap is small, shrink the number of bits per word so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * we spread over a few cachelines, at least. If less than 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * bits, just forget about it, it's not going to work optimally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) if (depth >= 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) while ((4U << shift) > depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) shift--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) bits_per_word = 1U << shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) if (bits_per_word > BITS_PER_LONG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) sb->shift = shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) sb->depth = depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) if (depth == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) sb->map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) sb->map = kcalloc_node(sb->map_nr, sizeof(*sb->map), flags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) if (!sb->map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) for (i = 0; i < sb->map_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) sb->map[i].depth = min(depth, bits_per_word);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) depth -= sb->map[i].depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) spin_lock_init(&sb->map[i].swap_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) EXPORT_SYMBOL_GPL(sbitmap_init_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) void sbitmap_resize(struct sbitmap *sb, unsigned int depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) unsigned int bits_per_word = 1U << sb->shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) for (i = 0; i < sb->map_nr; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) sbitmap_deferred_clear(sb, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) sb->depth = depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) for (i = 0; i < sb->map_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) sb->map[i].depth = min(depth, bits_per_word);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) depth -= sb->map[i].depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) EXPORT_SYMBOL_GPL(sbitmap_resize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) static int __sbitmap_get_word(unsigned long *word, unsigned long depth,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) unsigned int hint, bool wrap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) unsigned int orig_hint = hint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) int nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) nr = find_next_zero_bit(word, depth, hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (unlikely(nr >= depth)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * We started with an offset, and we didn't reset the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * offset to 0 in a failure case, so start from 0 to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * exhaust the map.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (orig_hint && hint && wrap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) hint = orig_hint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (!test_and_set_bit_lock(nr, word))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) hint = nr + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (hint >= depth - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) hint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) unsigned int alloc_hint, bool round_robin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) int nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) nr = __sbitmap_get_word(&sb->map[index].word,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) sb->map[index].depth, alloc_hint,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) !round_robin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (nr != -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (!sbitmap_deferred_clear(sb, index))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) } while (1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) return nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) unsigned int i, index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) int nr = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) index = SB_NR_TO_INDEX(sb, alloc_hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * Unless we're doing round robin tag allocation, just use the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * alloc_hint to find the right word index. No point in looping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * twice in find_next_zero_bit() for that case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (round_robin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) alloc_hint = SB_NR_TO_BIT(sb, alloc_hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) alloc_hint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) for (i = 0; i < sb->map_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) nr = sbitmap_find_bit_in_index(sb, index, alloc_hint,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) round_robin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (nr != -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) nr += index << sb->shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) /* Jump to next index. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) alloc_hint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (++index >= sb->map_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) return nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) EXPORT_SYMBOL_GPL(sbitmap_get);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) unsigned long shallow_depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) unsigned int i, index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) int nr = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) index = SB_NR_TO_INDEX(sb, alloc_hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) for (i = 0; i < sb->map_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) nr = __sbitmap_get_word(&sb->map[index].word,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) min(sb->map[index].depth, shallow_depth),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) SB_NR_TO_BIT(sb, alloc_hint), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) if (nr != -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) nr += index << sb->shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (sbitmap_deferred_clear(sb, index))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) /* Jump to next index. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) alloc_hint = index << sb->shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (index >= sb->map_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) alloc_hint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) return nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) EXPORT_SYMBOL_GPL(sbitmap_get_shallow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) bool sbitmap_any_bit_set(const struct sbitmap *sb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) for (i = 0; i < sb->map_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (sb->map[i].word & ~sb->map[i].cleared)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) EXPORT_SYMBOL_GPL(sbitmap_any_bit_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) static unsigned int __sbitmap_weight(const struct sbitmap *sb, bool set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) unsigned int i, weight = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) for (i = 0; i < sb->map_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) const struct sbitmap_word *word = &sb->map[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) weight += bitmap_weight(&word->word, word->depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) weight += bitmap_weight(&word->cleared, word->depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) static unsigned int sbitmap_weight(const struct sbitmap *sb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) return __sbitmap_weight(sb, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) static unsigned int sbitmap_cleared(const struct sbitmap *sb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) return __sbitmap_weight(sb, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) void sbitmap_show(struct sbitmap *sb, struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) seq_printf(m, "depth=%u\n", sb->depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) seq_printf(m, "busy=%u\n", sbitmap_weight(sb) - sbitmap_cleared(sb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) seq_printf(m, "cleared=%u\n", sbitmap_cleared(sb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) seq_printf(m, "bits_per_word=%u\n", 1U << sb->shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) seq_printf(m, "map_nr=%u\n", sb->map_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) EXPORT_SYMBOL_GPL(sbitmap_show);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) static inline void emit_byte(struct seq_file *m, unsigned int offset, u8 byte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if ((offset & 0xf) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (offset != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) seq_putc(m, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) seq_printf(m, "%08x:", offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if ((offset & 0x1) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) seq_putc(m, ' ');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) seq_printf(m, "%02x", byte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) u8 byte = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) unsigned int byte_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) unsigned int offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) for (i = 0; i < sb->map_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned long word = READ_ONCE(sb->map[i].word);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) unsigned long cleared = READ_ONCE(sb->map[i].cleared);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) unsigned int word_bits = READ_ONCE(sb->map[i].depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) word &= ~cleared;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) while (word_bits > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) unsigned int bits = min(8 - byte_bits, word_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) byte |= (word & (BIT(bits) - 1)) << byte_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) byte_bits += bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (byte_bits == 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) emit_byte(m, offset, byte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) byte = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) byte_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) offset++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) word >>= bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) word_bits -= bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (byte_bits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) emit_byte(m, offset, byte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) offset++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) seq_putc(m, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) EXPORT_SYMBOL_GPL(sbitmap_bitmap_show);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) static unsigned int sbq_calc_wake_batch(struct sbitmap_queue *sbq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) unsigned int depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) unsigned int wake_batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) unsigned int shallow_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * For each batch, we wake up one queue. We need to make sure that our
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * batch size is small enough that the full depth of the bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * potentially limited by a shallow depth, is enough to wake up all of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * the queues.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * Each full word of the bitmap has bits_per_word bits, and there might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * be a partial word. There are depth / bits_per_word full words and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * depth % bits_per_word bits left over. In bitwise arithmetic:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * bits_per_word = 1 << shift
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * depth / bits_per_word = depth >> shift
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * depth % bits_per_word = depth & ((1 << shift) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * Each word can be limited to sbq->min_shallow_depth bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) shallow_depth = min(1U << sbq->sb.shift, sbq->min_shallow_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) depth = ((depth >> sbq->sb.shift) * shallow_depth +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) wake_batch = clamp_t(unsigned int, depth / SBQ_WAIT_QUEUES, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) SBQ_WAKE_BATCH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) return wake_batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) int shift, bool round_robin, gfp_t flags, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) sbq->alloc_hint = alloc_percpu_gfp(unsigned int, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) if (!sbq->alloc_hint) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) sbitmap_free(&sbq->sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (depth && !round_robin) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) for_each_possible_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) *per_cpu_ptr(sbq->alloc_hint, i) = prandom_u32() % depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) sbq->min_shallow_depth = UINT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) sbq->wake_batch = sbq_calc_wake_batch(sbq, depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) atomic_set(&sbq->wake_index, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) atomic_set(&sbq->ws_active, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) sbq->ws = kzalloc_node(SBQ_WAIT_QUEUES * sizeof(*sbq->ws), flags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (!sbq->ws) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) free_percpu(sbq->alloc_hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) sbitmap_free(&sbq->sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) for (i = 0; i < SBQ_WAIT_QUEUES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) init_waitqueue_head(&sbq->ws[i].wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) atomic_set(&sbq->ws[i].wait_cnt, sbq->wake_batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) sbq->round_robin = round_robin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) EXPORT_SYMBOL_GPL(sbitmap_queue_init_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) static void sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) unsigned int depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) unsigned int wake_batch = sbq_calc_wake_batch(sbq, depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (sbq->wake_batch != wake_batch) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) WRITE_ONCE(sbq->wake_batch, wake_batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) * Pairs with the memory barrier in sbitmap_queue_wake_up()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) * to ensure that the batch size is updated before the wait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * counts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) for (i = 0; i < SBQ_WAIT_QUEUES; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) atomic_set(&sbq->ws[i].wait_cnt, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) sbitmap_queue_update_wake_batch(sbq, depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) sbitmap_resize(&sbq->sb, depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) EXPORT_SYMBOL_GPL(sbitmap_queue_resize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) int __sbitmap_queue_get(struct sbitmap_queue *sbq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) unsigned int hint, depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) int nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) hint = this_cpu_read(*sbq->alloc_hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) depth = READ_ONCE(sbq->sb.depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (unlikely(hint >= depth)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) hint = depth ? prandom_u32() % depth : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) this_cpu_write(*sbq->alloc_hint, hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) nr = sbitmap_get(&sbq->sb, hint, sbq->round_robin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) if (nr == -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* If the map is full, a hint won't do us much good. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) this_cpu_write(*sbq->alloc_hint, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) } else if (nr == hint || unlikely(sbq->round_robin)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) /* Only update the hint if we used it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) hint = nr + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (hint >= depth - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) hint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) this_cpu_write(*sbq->alloc_hint, hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) EXPORT_SYMBOL_GPL(__sbitmap_queue_get);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) unsigned int shallow_depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) unsigned int hint, depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) int nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) WARN_ON_ONCE(shallow_depth < sbq->min_shallow_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) hint = this_cpu_read(*sbq->alloc_hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) depth = READ_ONCE(sbq->sb.depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) if (unlikely(hint >= depth)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) hint = depth ? prandom_u32() % depth : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) this_cpu_write(*sbq->alloc_hint, hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) nr = sbitmap_get_shallow(&sbq->sb, hint, shallow_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (nr == -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) /* If the map is full, a hint won't do us much good. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) this_cpu_write(*sbq->alloc_hint, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) } else if (nr == hint || unlikely(sbq->round_robin)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) /* Only update the hint if we used it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) hint = nr + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (hint >= depth - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) hint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) this_cpu_write(*sbq->alloc_hint, hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) return nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) EXPORT_SYMBOL_GPL(__sbitmap_queue_get_shallow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) void sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) unsigned int min_shallow_depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) sbq->min_shallow_depth = min_shallow_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) sbitmap_queue_update_wake_batch(sbq, sbq->sb.depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) EXPORT_SYMBOL_GPL(sbitmap_queue_min_shallow_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) int i, wake_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) if (!atomic_read(&sbq->ws_active))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) wake_index = atomic_read(&sbq->wake_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) for (i = 0; i < SBQ_WAIT_QUEUES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) struct sbq_wait_state *ws = &sbq->ws[wake_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (waitqueue_active(&ws->wait)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (wake_index != atomic_read(&sbq->wake_index))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) atomic_set(&sbq->wake_index, wake_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) return ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) wake_index = sbq_index_inc(wake_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) static bool __sbq_wake_up(struct sbitmap_queue *sbq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) struct sbq_wait_state *ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) unsigned int wake_batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) int wait_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) ws = sbq_wake_ptr(sbq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (!ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) wait_cnt = atomic_dec_return(&ws->wait_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) if (wait_cnt <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) wake_batch = READ_ONCE(sbq->wake_batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * Pairs with the memory barrier in sbitmap_queue_resize() to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * ensure that we see the batch size update before the wait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * count is reset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * For concurrent callers of this, the one that failed the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * atomic_cmpxhcg() race should call this function again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * to wakeup a new batch on a different 'ws'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) ret = atomic_cmpxchg(&ws->wait_cnt, wait_cnt, wake_batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (ret == wait_cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) sbq_index_atomic_inc(&sbq->wake_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) wake_up_nr(&ws->wait, wake_batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) void sbitmap_queue_wake_up(struct sbitmap_queue *sbq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) while (__sbq_wake_up(sbq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) EXPORT_SYMBOL_GPL(sbitmap_queue_wake_up);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * Once the clear bit is set, the bit may be allocated out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * Orders READ/WRITE on the asssociated instance(such as request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * of blk_mq) by this bit for avoiding race with re-allocation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * and its pair is the memory barrier implied in __sbitmap_get_word.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * One invariant is that the clear bit has to be zero when the bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * is in use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) sbitmap_deferred_clear_bit(&sbq->sb, nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) * Pairs with the memory barrier in set_current_state() to ensure the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) * proper ordering of clear_bit_unlock()/waitqueue_active() in the waker
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * and test_and_set_bit_lock()/prepare_to_wait()/finish_wait() in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * waiter. See the comment on waitqueue_active().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) sbitmap_queue_wake_up(sbq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (likely(!sbq->round_robin && nr < sbq->sb.depth))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) *per_cpu_ptr(sbq->alloc_hint, cpu) = nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) EXPORT_SYMBOL_GPL(sbitmap_queue_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) void sbitmap_queue_wake_all(struct sbitmap_queue *sbq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) int i, wake_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) * Pairs with the memory barrier in set_current_state() like in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) * sbitmap_queue_wake_up().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) wake_index = atomic_read(&sbq->wake_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) for (i = 0; i < SBQ_WAIT_QUEUES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) struct sbq_wait_state *ws = &sbq->ws[wake_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) if (waitqueue_active(&ws->wait))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) wake_up(&ws->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) wake_index = sbq_index_inc(wake_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) EXPORT_SYMBOL_GPL(sbitmap_queue_wake_all);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) bool first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) sbitmap_show(&sbq->sb, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) seq_puts(m, "alloc_hint={");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) first = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) for_each_possible_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) if (!first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) seq_puts(m, ", ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) first = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) seq_printf(m, "%u", *per_cpu_ptr(sbq->alloc_hint, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) seq_puts(m, "}\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) seq_printf(m, "wake_batch=%u\n", sbq->wake_batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) seq_printf(m, "wake_index=%d\n", atomic_read(&sbq->wake_index));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) seq_printf(m, "ws_active=%d\n", atomic_read(&sbq->ws_active));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) seq_puts(m, "ws={\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) for (i = 0; i < SBQ_WAIT_QUEUES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) struct sbq_wait_state *ws = &sbq->ws[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) seq_printf(m, "\t{.wait_cnt=%d, .wait=%s},\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) atomic_read(&ws->wait_cnt),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) waitqueue_active(&ws->wait) ? "active" : "inactive");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) seq_puts(m, "}\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) seq_printf(m, "round_robin=%d\n", sbq->round_robin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) seq_printf(m, "min_shallow_depth=%u\n", sbq->min_shallow_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) EXPORT_SYMBOL_GPL(sbitmap_queue_show);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) void sbitmap_add_wait_queue(struct sbitmap_queue *sbq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) struct sbq_wait_state *ws,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) struct sbq_wait *sbq_wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) if (!sbq_wait->sbq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) sbq_wait->sbq = sbq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) atomic_inc(&sbq->ws_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) add_wait_queue(&ws->wait, &sbq_wait->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) EXPORT_SYMBOL_GPL(sbitmap_add_wait_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) void sbitmap_del_wait_queue(struct sbq_wait *sbq_wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) list_del_init(&sbq_wait->wait.entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) if (sbq_wait->sbq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) atomic_dec(&sbq_wait->sbq->ws_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) sbq_wait->sbq = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) EXPORT_SYMBOL_GPL(sbitmap_del_wait_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) struct sbq_wait_state *ws,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) struct sbq_wait *sbq_wait, int state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) if (!sbq_wait->sbq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) atomic_inc(&sbq->ws_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) sbq_wait->sbq = sbq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) prepare_to_wait_exclusive(&ws->wait, &sbq_wait->wait, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) EXPORT_SYMBOL_GPL(sbitmap_prepare_to_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) struct sbq_wait *sbq_wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) finish_wait(&ws->wait, &sbq_wait->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (sbq_wait->sbq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) atomic_dec(&sbq->ws_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) sbq_wait->sbq = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) EXPORT_SYMBOL_GPL(sbitmap_finish_wait);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags