^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #ifndef _BCACHE_UTIL_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #define _BCACHE_UTIL_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/sched/clock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/llist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/ratelimit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/crc64.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include "closure.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define PAGE_SECTORS (PAGE_SIZE / 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) struct closure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #ifdef CONFIG_BCACHE_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define EBUG_ON(cond) BUG_ON(cond)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define atomic_dec_bug(v) BUG_ON(atomic_dec_return(v) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define atomic_inc_bug(v, i) BUG_ON(atomic_inc_return(v) <= i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #else /* DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define EBUG_ON(cond) do { if (cond); } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define atomic_dec_bug(v) atomic_dec(v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define atomic_inc_bug(v, i) atomic_inc(v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define DECLARE_HEAP(type, name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) size_t size, used; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) type *data; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) } name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define init_heap(heap, _size, gfp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) size_t _bytes; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) (heap)->used = 0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) (heap)->size = (_size); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) _bytes = (heap)->size * sizeof(*(heap)->data); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) (heap)->data = kvmalloc(_bytes, (gfp) & GFP_KERNEL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) (heap)->data; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define free_heap(heap) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) kvfree((heap)->data); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) (heap)->data = NULL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define heap_swap(h, i, j) swap((h)->data[i], (h)->data[j])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define heap_sift(h, i, cmp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) size_t _r, _j = i; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) for (; _j * 2 + 1 < (h)->used; _j = _r) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) _r = _j * 2 + 1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (_r + 1 < (h)->used && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) cmp((h)->data[_r], (h)->data[_r + 1])) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) _r++; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) if (cmp((h)->data[_r], (h)->data[_j])) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) heap_swap(h, _r, _j); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define heap_sift_down(h, i, cmp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) while (i) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) size_t p = (i - 1) / 2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) if (cmp((h)->data[i], (h)->data[p])) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) heap_swap(h, i, p); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) i = p; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define heap_add(h, d, cmp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) bool _r = !heap_full(h); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (_r) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) size_t _i = (h)->used++; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) (h)->data[_i] = d; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) heap_sift_down(h, _i, cmp); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) heap_sift(h, _i, cmp); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) _r; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define heap_pop(h, d, cmp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) bool _r = (h)->used; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (_r) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) (d) = (h)->data[0]; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) (h)->used--; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) heap_swap(h, 0, (h)->used); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) heap_sift(h, 0, cmp); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) _r; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #define heap_peek(h) ((h)->used ? (h)->data[0] : NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #define heap_full(h) ((h)->used == (h)->size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define DECLARE_FIFO(type, name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) size_t front, back, size, mask; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) type *data; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) } name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #define fifo_for_each(c, fifo, iter) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) for (iter = (fifo)->front; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) c = (fifo)->data[iter], iter != (fifo)->back; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) iter = (iter + 1) & (fifo)->mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #define __init_fifo(fifo, gfp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) size_t _allocated_size, _bytes; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) BUG_ON(!(fifo)->size); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) _allocated_size = roundup_pow_of_two((fifo)->size + 1); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) _bytes = _allocated_size * sizeof(*(fifo)->data); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) (fifo)->mask = _allocated_size - 1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) (fifo)->front = (fifo)->back = 0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) (fifo)->data = kvmalloc(_bytes, (gfp) & GFP_KERNEL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) (fifo)->data; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define init_fifo_exact(fifo, _size, gfp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) (fifo)->size = (_size); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) __init_fifo(fifo, gfp); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #define init_fifo(fifo, _size, gfp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) (fifo)->size = (_size); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if ((fifo)->size > 4) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) (fifo)->size = roundup_pow_of_two((fifo)->size) - 1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) __init_fifo(fifo, gfp); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #define free_fifo(fifo) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) kvfree((fifo)->data); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) (fifo)->data = NULL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #define fifo_used(fifo) (((fifo)->back - (fifo)->front) & (fifo)->mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) #define fifo_free(fifo) ((fifo)->size - fifo_used(fifo))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #define fifo_empty(fifo) (!fifo_used(fifo))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #define fifo_full(fifo) (!fifo_free(fifo))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) #define fifo_front(fifo) ((fifo)->data[(fifo)->front])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #define fifo_back(fifo) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) ((fifo)->data[((fifo)->back - 1) & (fifo)->mask])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) #define fifo_idx(fifo, p) (((p) - &fifo_front(fifo)) & (fifo)->mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) #define fifo_push_back(fifo, i) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) bool _r = !fifo_full((fifo)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (_r) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) (fifo)->data[(fifo)->back++] = (i); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) (fifo)->back &= (fifo)->mask; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) _r; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #define fifo_pop_front(fifo, i) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) bool _r = !fifo_empty((fifo)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (_r) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) (i) = (fifo)->data[(fifo)->front++]; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) (fifo)->front &= (fifo)->mask; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) _r; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #define fifo_push_front(fifo, i) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) bool _r = !fifo_full((fifo)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (_r) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) --(fifo)->front; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) (fifo)->front &= (fifo)->mask; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) (fifo)->data[(fifo)->front] = (i); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) _r; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) #define fifo_pop_back(fifo, i) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) bool _r = !fifo_empty((fifo)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (_r) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) --(fifo)->back; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) (fifo)->back &= (fifo)->mask; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) (i) = (fifo)->data[(fifo)->back] \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) _r; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) #define fifo_push(fifo, i) fifo_push_back(fifo, (i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) #define fifo_pop(fifo, i) fifo_pop_front(fifo, (i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #define fifo_swap(l, r) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) swap((l)->front, (r)->front); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) swap((l)->back, (r)->back); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) swap((l)->size, (r)->size); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) swap((l)->mask, (r)->mask); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) swap((l)->data, (r)->data); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) #define fifo_move(dest, src) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) typeof(*((dest)->data)) _t; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) while (!fifo_full(dest) && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) fifo_pop(src, _t)) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) fifo_push(dest, _t); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * Simple array based allocator - preallocates a number of elements and you can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * never allocate more than that, also has no locking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * Handy because if you know you only need a fixed number of elements you don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * have to worry about memory allocation failure, and sometimes a mempool isn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * what you want.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * We treat the free elements as entries in a singly linked list, and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * freelist as a stack - allocating and freeing push and pop off the freelist.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) #define DECLARE_ARRAY_ALLOCATOR(type, name, size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) type *freelist; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) type data[size]; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) } name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) #define array_alloc(array) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) typeof((array)->freelist) _ret = (array)->freelist; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (_ret) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) (array)->freelist = *((typeof((array)->freelist) *) _ret);\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) _ret; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) #define array_free(array, ptr) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) typeof((array)->freelist) _ptr = ptr; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) *((typeof((array)->freelist) *) _ptr) = (array)->freelist; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) (array)->freelist = _ptr; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #define array_allocator_init(array) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) typeof((array)->freelist) _i; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) BUILD_BUG_ON(sizeof((array)->data[0]) < sizeof(void *)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) (array)->freelist = NULL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) for (_i = (array)->data; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) _i < (array)->data + ARRAY_SIZE((array)->data); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) _i++) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) array_free(array, _i); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) #define array_freelist_empty(array) ((array)->freelist == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) #define ANYSINT_MAX(t) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) ((((t) 1 << (sizeof(t) * 8 - 2)) - (t) 1) * (t) 2 + (t) 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) int bch_strtoint_h(const char *cp, int *res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) int bch_strtouint_h(const char *cp, unsigned int *res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) int bch_strtoll_h(const char *cp, long long *res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) int bch_strtoull_h(const char *cp, unsigned long long *res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) static inline int bch_strtol_h(const char *cp, long *res)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) #if BITS_PER_LONG == 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return bch_strtoint_h(cp, (int *) res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) return bch_strtoll_h(cp, (long long *) res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static inline int bch_strtoul_h(const char *cp, long *res)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) #if BITS_PER_LONG == 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return bch_strtouint_h(cp, (unsigned int *) res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) return bch_strtoull_h(cp, (unsigned long long *) res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) #define strtoi_h(cp, res) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) (__builtin_types_compatible_p(typeof(*res), int) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) ? bch_strtoint_h(cp, (void *) res) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) : __builtin_types_compatible_p(typeof(*res), long) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) ? bch_strtol_h(cp, (void *) res) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) : __builtin_types_compatible_p(typeof(*res), long long) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) ? bch_strtoll_h(cp, (void *) res) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) : __builtin_types_compatible_p(typeof(*res), unsigned int) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) ? bch_strtouint_h(cp, (void *) res) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) : __builtin_types_compatible_p(typeof(*res), unsigned long) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) ? bch_strtoul_h(cp, (void *) res) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) : __builtin_types_compatible_p(typeof(*res), unsigned long long)\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) ? bch_strtoull_h(cp, (void *) res) : -EINVAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) #define strtoul_safe(cp, var) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) unsigned long _v; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) int _r = kstrtoul(cp, 10, &_v); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) if (!_r) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) var = _v; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) _r; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) #define strtoul_safe_clamp(cp, var, min, max) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) unsigned long _v; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) int _r = kstrtoul(cp, 10, &_v); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (!_r) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) var = clamp_t(typeof(var), _v, min, max); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) _r; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) #define snprint(buf, size, var) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) snprintf(buf, size, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) __builtin_types_compatible_p(typeof(var), int) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) ? "%i\n" : \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) __builtin_types_compatible_p(typeof(var), unsigned int) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) ? "%u\n" : \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) __builtin_types_compatible_p(typeof(var), long) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) ? "%li\n" : \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) __builtin_types_compatible_p(typeof(var), unsigned long)\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) ? "%lu\n" : \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) __builtin_types_compatible_p(typeof(var), int64_t) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) ? "%lli\n" : \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) __builtin_types_compatible_p(typeof(var), uint64_t) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) ? "%llu\n" : \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) __builtin_types_compatible_p(typeof(var), const char *) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) ? "%s\n" : "%i\n", var)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) ssize_t bch_hprint(char *buf, int64_t v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) bool bch_is_zero(const char *p, size_t n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) int bch_parse_uuid(const char *s, char *uuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct time_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * all fields are in nanoseconds, averages are ewmas stored left shifted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) uint64_t max_duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) uint64_t average_duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) uint64_t average_frequency;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) uint64_t last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) void bch_time_stats_update(struct time_stats *stats, uint64_t time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) static inline unsigned int local_clock_us(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return local_clock() >> 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) #define NSEC_PER_ns 1L
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) #define NSEC_PER_us NSEC_PER_USEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) #define NSEC_PER_ms NSEC_PER_MSEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) #define NSEC_PER_sec NSEC_PER_SEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) #define __print_time_stat(stats, name, stat, units) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) sysfs_print(name ## _ ## stat ## _ ## units, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) div_u64((stats)->stat >> 8, NSEC_PER_ ## units))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) #define sysfs_print_time_stats(stats, name, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) frequency_units, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) duration_units) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) __print_time_stat(stats, name, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) average_frequency, frequency_units); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) __print_time_stat(stats, name, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) average_duration, duration_units); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) sysfs_print(name ## _ ##max_duration ## _ ## duration_units, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) div_u64((stats)->max_duration, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) NSEC_PER_ ## duration_units)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) sysfs_print(name ## _last_ ## frequency_units, (stats)->last \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) ? div_s64(local_clock() - (stats)->last, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) NSEC_PER_ ## frequency_units) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) : -1LL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) #define sysfs_time_stats_attribute(name, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) frequency_units, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) duration_units) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) read_attribute(name ## _average_frequency_ ## frequency_units); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) read_attribute(name ## _average_duration_ ## duration_units); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) read_attribute(name ## _max_duration_ ## duration_units); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) read_attribute(name ## _last_ ## frequency_units)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) #define sysfs_time_stats_attribute_list(name, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) frequency_units, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) duration_units) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) &sysfs_ ## name ## _average_frequency_ ## frequency_units, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) &sysfs_ ## name ## _average_duration_ ## duration_units, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) &sysfs_ ## name ## _max_duration_ ## duration_units, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) &sysfs_ ## name ## _last_ ## frequency_units,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) #define ewma_add(ewma, val, weight, factor) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) (ewma) *= (weight) - 1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) (ewma) += (val) << factor; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) (ewma) /= (weight); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) (ewma) >> factor; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct bch_ratelimit {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* Next time we want to do some work, in nanoseconds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) uint64_t next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) * Rate at which we want to do work, in units per second
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * The units here correspond to the units passed to bch_next_delay()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) atomic_long_t rate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) static inline void bch_ratelimit_reset(struct bch_ratelimit *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) d->next = local_clock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) uint64_t bch_next_delay(struct bch_ratelimit *d, uint64_t done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) #define __DIV_SAFE(n, d, zero) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) typeof(n) _n = (n); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) typeof(d) _d = (d); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) _d ? _n / _d : zero; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) #define DIV_SAFE(n, d) __DIV_SAFE(n, d, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) #define container_of_or_null(ptr, type, member) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) typeof(ptr) _ptr = ptr; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) _ptr ? container_of(_ptr, type, member) : NULL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) #define RB_INSERT(root, new, member, cmp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) __label__ dup; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) struct rb_node **n = &(root)->rb_node, *parent = NULL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) typeof(new) this; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) int res, ret = -1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) while (*n) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) parent = *n; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) this = container_of(*n, typeof(*(new)), member); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) res = cmp(new, this); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) if (!res) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) goto dup; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) n = res < 0 \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) ? &(*n)->rb_left \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) : &(*n)->rb_right; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) rb_link_node(&(new)->member, parent, n); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) rb_insert_color(&(new)->member, root); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) ret = 0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) dup: \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) ret; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) #define RB_SEARCH(root, search, member, cmp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) struct rb_node *n = (root)->rb_node; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) typeof(&(search)) this, ret = NULL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) int res; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) while (n) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) this = container_of(n, typeof(search), member); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) res = cmp(&(search), this); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (!res) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) ret = this; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) n = res < 0 \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) ? n->rb_left \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) : n->rb_right; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) ret; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) #define RB_GREATER(root, search, member, cmp) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) struct rb_node *n = (root)->rb_node; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) typeof(&(search)) this, ret = NULL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) int res; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) while (n) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) this = container_of(n, typeof(search), member); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) res = cmp(&(search), this); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (res < 0) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) ret = this; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) n = n->rb_left; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) } else \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) n = n->rb_right; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) ret; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) #define RB_FIRST(root, type, member) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) container_of_or_null(rb_first(root), type, member)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) #define RB_LAST(root, type, member) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) container_of_or_null(rb_last(root), type, member)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) #define RB_NEXT(ptr, member) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) container_of_or_null(rb_next(&(ptr)->member), typeof(*ptr), member)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) #define RB_PREV(ptr, member) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) container_of_or_null(rb_prev(&(ptr)->member), typeof(*ptr), member)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) static inline uint64_t bch_crc64(const void *p, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) uint64_t crc = 0xffffffffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) crc = crc64_be(crc, p, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) return crc ^ 0xffffffffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) static inline uint64_t bch_crc64_update(uint64_t crc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) const void *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) crc = crc64_be(crc, p, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) return crc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * A stepwise-linear pseudo-exponential. This returns 1 << (x >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * frac_bits), with the less-significant bits filled in by linear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) * interpolation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * This can also be interpreted as a floating-point number format,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * where the low frac_bits are the mantissa (with implicit leading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * 1 bit), and the more significant bits are the exponent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * The return value is 1.mantissa * 2^exponent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) * The way this is used, fract_bits is 6 and the largest possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * input is CONGESTED_MAX-1 = 1023 (exponent 16, mantissa 0x1.fc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * so the maximum output is 0x1fc00.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) static inline unsigned int fract_exp_two(unsigned int x,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) unsigned int fract_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) unsigned int mantissa = 1 << fract_bits; /* Implicit bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) mantissa += x & (mantissa - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) x >>= fract_bits; /* The exponent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /* Largest intermediate value 0x7f0000 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) return mantissa << x >> fract_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) void bch_bio_map(struct bio *bio, void *base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) int bch_bio_alloc_pages(struct bio *bio, gfp_t gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) static inline sector_t bdev_sectors(struct block_device *bdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) return bdev->bd_inode->i_size >> 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) #endif /* _BCACHE_UTIL_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags