^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/mm/mempool.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * memory buffer pool support. Such pools are mostly used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * for guaranteed, deadlock-free memory allocations during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * extreme VM load.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * started by Ingo Molnar, Copyright (C) 2001
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * debugging by David Rientjes, Copyright (C) 2015
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/kmemleak.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/writeback.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "slab.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB_DEBUG_ON)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) static void poison_error(mempool_t *pool, void *element, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) size_t byte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) const int nr = pool->curr_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) const int start = max_t(int, byte - (BITS_PER_LONG / 8), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) const int end = min_t(int, byte + (BITS_PER_LONG / 8), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) pr_err("BUG: mempool element poison mismatch\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) pr_err("Mempool %p size %zu\n", pool, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) for (i = start; i < end; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) pr_cont("%x ", *(u8 *)(element + i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) pr_cont("%s\n", end < size ? "..." : "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static void __check_element(mempool_t *pool, void *element, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) u8 *obj = element;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) size_t i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) for (i = 0; i < size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) u8 exp = (i < size - 1) ? POISON_FREE : POISON_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) if (obj[i] != exp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) poison_error(pool, element, size, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) memset(obj, POISON_INUSE, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static void check_element(mempool_t *pool, void *element)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /* Mempools backed by slab allocator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) if (pool->free == mempool_free_slab || pool->free == mempool_kfree) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) __check_element(pool, element, ksize(element));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) } else if (pool->free == mempool_free_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) /* Mempools backed by page allocator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) int order = (int)(long)pool->pool_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) void *addr = kmap_atomic((struct page *)element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) __check_element(pool, addr, 1UL << (PAGE_SHIFT + order));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) kunmap_atomic(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static void __poison_element(void *element, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) u8 *obj = element;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) memset(obj, POISON_FREE, size - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) obj[size - 1] = POISON_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static void poison_element(mempool_t *pool, void *element)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* Mempools backed by slab allocator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) __poison_element(element, ksize(element));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) } else if (pool->alloc == mempool_alloc_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /* Mempools backed by page allocator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) int order = (int)(long)pool->pool_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) void *addr = kmap_atomic((struct page *)element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) __poison_element(addr, 1UL << (PAGE_SHIFT + order));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) kunmap_atomic(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #else /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static inline void check_element(mempool_t *pool, void *element)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static inline void poison_element(mempool_t *pool, void *element)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #endif /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) static __always_inline void kasan_poison_element(mempool_t *pool, void *element)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) kasan_slab_free_mempool(element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) else if (pool->alloc == mempool_alloc_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) kasan_poison_pages(element, (unsigned long)pool->pool_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) static void kasan_unpoison_element(mempool_t *pool, void *element)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) kasan_unpoison_range(element, __ksize(element));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) else if (pool->alloc == mempool_alloc_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) kasan_unpoison_pages(element, (unsigned long)pool->pool_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) static __always_inline void add_element(mempool_t *pool, void *element)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) BUG_ON(pool->curr_nr >= pool->min_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) poison_element(pool, element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) kasan_poison_element(pool, element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) pool->elements[pool->curr_nr++] = element;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static void *remove_element(mempool_t *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) void *element = pool->elements[--pool->curr_nr];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) BUG_ON(pool->curr_nr < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) kasan_unpoison_element(pool, element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) check_element(pool, element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return element;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * mempool_exit - exit a mempool initialized with mempool_init()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * @pool: pointer to the memory pool which was initialized with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * mempool_init().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * Free all reserved elements in @pool and @pool itself. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * only sleeps if the free_fn() function sleeps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * May be called on a zeroed but uninitialized mempool (i.e. allocated with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * kzalloc()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) void mempool_exit(mempool_t *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) while (pool->curr_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) void *element = remove_element(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) pool->free(element, pool->pool_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) kfree(pool->elements);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) pool->elements = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) EXPORT_SYMBOL(mempool_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * mempool_destroy - deallocate a memory pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * @pool: pointer to the memory pool which was allocated via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * mempool_create().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * Free all reserved elements in @pool and @pool itself. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * only sleeps if the free_fn() function sleeps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) void mempool_destroy(mempool_t *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (unlikely(!pool))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) mempool_exit(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) kfree(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) EXPORT_SYMBOL(mempool_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) mempool_free_t *free_fn, void *pool_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) gfp_t gfp_mask, int node_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) spin_lock_init(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) pool->min_nr = min_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) pool->pool_data = pool_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) pool->alloc = alloc_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) pool->free = free_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) init_waitqueue_head(&pool->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) pool->elements = kmalloc_array_node(min_nr, sizeof(void *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) gfp_mask, node_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (!pool->elements)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * First pre-allocate the guaranteed number of buffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) while (pool->curr_nr < pool->min_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) void *element;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) element = pool->alloc(gfp_mask, pool->pool_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (unlikely(!element)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) mempool_exit(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) add_element(pool, element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) EXPORT_SYMBOL(mempool_init_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * mempool_init - initialize a memory pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * @pool: pointer to the memory pool that should be initialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * @min_nr: the minimum number of elements guaranteed to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * allocated for this pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * @alloc_fn: user-defined element-allocation function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * @free_fn: user-defined element-freeing function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * @pool_data: optional private data available to the user-defined functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * Like mempool_create(), but initializes the pool in (i.e. embedded in another
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * structure).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * Return: %0 on success, negative error code otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) mempool_free_t *free_fn, void *pool_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) pool_data, GFP_KERNEL, NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) EXPORT_SYMBOL(mempool_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * mempool_create - create a memory pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * @min_nr: the minimum number of elements guaranteed to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * allocated for this pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * @alloc_fn: user-defined element-allocation function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * @free_fn: user-defined element-freeing function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * @pool_data: optional private data available to the user-defined functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * this function creates and allocates a guaranteed size, preallocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * functions. This function might sleep. Both the alloc_fn() and the free_fn()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * functions might sleep - as long as the mempool_alloc() function is not called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * from IRQ contexts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * Return: pointer to the created memory pool object or %NULL on error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) mempool_free_t *free_fn, void *pool_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) GFP_KERNEL, NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) EXPORT_SYMBOL(mempool_create);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) mempool_free_t *free_fn, void *pool_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) gfp_t gfp_mask, int node_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) mempool_t *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (!pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) gfp_mask, node_id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) kfree(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) EXPORT_SYMBOL(mempool_create_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * mempool_resize - resize an existing memory pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * @pool: pointer to the memory pool which was allocated via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) * mempool_create().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * @new_min_nr: the new minimum number of elements guaranteed to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * allocated for this pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * This function shrinks/grows the pool. In the case of growing,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * it cannot be guaranteed that the pool will be grown to the new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * size immediately, but new mempool_free() calls will refill it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * This function may sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * Note, the caller must guarantee that no mempool_destroy is called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * while this function is running. mempool_alloc() & mempool_free()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * might be called (eg. from IRQ contexts) while this function executes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * Return: %0 on success, negative error code otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) int mempool_resize(mempool_t *pool, int new_min_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) void *element;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) void **new_elements;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) BUG_ON(new_min_nr <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (new_min_nr <= pool->min_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) while (new_min_nr < pool->curr_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) element = remove_element(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) pool->free(element, pool->pool_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) pool->min_nr = new_min_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /* Grow the pool */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) new_elements = kmalloc_array(new_min_nr, sizeof(*new_elements),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (!new_elements)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (unlikely(new_min_nr <= pool->min_nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) /* Raced, other resize will do our work */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) kfree(new_elements);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) memcpy(new_elements, pool->elements,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) pool->curr_nr * sizeof(*new_elements));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) kfree(pool->elements);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) pool->elements = new_elements;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) pool->min_nr = new_min_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) while (pool->curr_nr < pool->min_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) element = pool->alloc(GFP_KERNEL, pool->pool_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (!element)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) if (pool->curr_nr < pool->min_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) add_element(pool, element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) pool->free(element, pool->pool_data); /* Raced */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) EXPORT_SYMBOL(mempool_resize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * mempool_alloc - allocate an element from a specific memory pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * @pool: pointer to the memory pool which was allocated via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) * mempool_create().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * @gfp_mask: the usual allocation bitmask.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * this function only sleeps if the alloc_fn() function sleeps or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * returns NULL. Note that due to preallocation, this function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * *never* fails when called from process contexts. (it might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * fail if called from an IRQ context.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * Note: using __GFP_ZERO is not supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * Return: pointer to the allocated element or %NULL on error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) void *element;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) wait_queue_entry_t wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) gfp_t gfp_temp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) gfp_mask |= __GFP_NOWARN; /* failures are OK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) repeat_alloc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) element = pool->alloc(gfp_temp, pool->pool_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) if (likely(element != NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) return element;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (likely(pool->curr_nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) element = remove_element(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) /* paired with rmb in mempool_free(), read comment there */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * Update the allocation stack trace as this is more useful
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * for debugging.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) kmemleak_update_trace(element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) return element;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * We use gfp mask w/o direct reclaim or IO for the first round. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * alloc failed with that and @pool was empty, retry immediately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (gfp_temp != gfp_mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) gfp_temp = gfp_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) goto repeat_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) /* We must not sleep if !__GFP_DIRECT_RECLAIM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) /* Let's wait for someone else to return an element to @pool */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) init_wait(&wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * FIXME: this should be io_schedule(). The timeout is there as a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * workaround for some DM problems in 2.6.18.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) io_schedule_timeout(5*HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) finish_wait(&pool->wait, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) goto repeat_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) EXPORT_SYMBOL(mempool_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * mempool_free - return an element to the pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * @element: pool element pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * @pool: pointer to the memory pool which was allocated via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * mempool_create().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * this function only sleeps if the free_fn() function sleeps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) void mempool_free(void *element, mempool_t *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (unlikely(element == NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * Paired with the wmb in mempool_alloc(). The preceding read is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * for @element and the following @pool->curr_nr. This ensures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * that the visible value of @pool->curr_nr is from after the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * allocation of @element. This is necessary for fringe cases
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * where @element was passed to this task without going through
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * barriers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * For example, assume @p is %NULL at the beginning and one task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * performs "p = mempool_alloc(...);" while another task is doing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * "while (!p) cpu_relax(); mempool_free(p, ...);". This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * may end up using curr_nr value which is from before allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * of @p without the following rmb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) * For correctness, we need a test which is guaranteed to trigger
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * if curr_nr + #allocated == min_nr. Testing curr_nr < min_nr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * without locking achieves that and refilling as soon as possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * is desirable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * Because curr_nr visible here is always a value after the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * allocation of @element, any task which decremented curr_nr below
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) * incremented to min_nr afterwards. If curr_nr gets incremented
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) * to min_nr after the allocation of @element, the elements
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * allocated after that are subject to the same guarantee.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * Waiters happen iff curr_nr is 0 and the above guarantee also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * ensures that there will be frees which return elements to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * pool waking up the waiters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (likely(pool->curr_nr < pool->min_nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) add_element(pool, element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) wake_up(&pool->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) pool->free(element, pool->pool_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) EXPORT_SYMBOL(mempool_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * A commonly used alloc and free fn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) struct kmem_cache *mem = pool_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) VM_BUG_ON(mem->ctor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) return kmem_cache_alloc(mem, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) EXPORT_SYMBOL(mempool_alloc_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) void mempool_free_slab(void *element, void *pool_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) struct kmem_cache *mem = pool_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) kmem_cache_free(mem, element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) EXPORT_SYMBOL(mempool_free_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * specified by pool_data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) size_t size = (size_t)pool_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return kmalloc(size, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) EXPORT_SYMBOL(mempool_kmalloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) void mempool_kfree(void *element, void *pool_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) kfree(element);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) EXPORT_SYMBOL(mempool_kfree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * A simple mempool-backed page allocator that allocates pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * of the order specified by pool_data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) int order = (int)(long)pool_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return alloc_pages(gfp_mask, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) EXPORT_SYMBOL(mempool_alloc_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) void mempool_free_pages(void *element, void *pool_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) int order = (int)(long)pool_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) __free_pages(element, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) EXPORT_SYMBOL(mempool_free_pages);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags