^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/scatterlist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #define SG_MEMPOOL_NR ARRAY_SIZE(sg_pools)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #define SG_MEMPOOL_SIZE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) struct sg_pool {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) struct kmem_cache *slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) mempool_t *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define SP(x) { .size = x, "sgpool-" __stringify(x) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #if (SG_CHUNK_SIZE < 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #error SG_CHUNK_SIZE is too small (must be 32 or greater)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) static struct sg_pool sg_pools[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) SP(8),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) SP(16),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #if (SG_CHUNK_SIZE > 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) SP(32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #if (SG_CHUNK_SIZE > 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) SP(64),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #if (SG_CHUNK_SIZE > 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) SP(128),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #if (SG_CHUNK_SIZE > 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #error SG_CHUNK_SIZE is too large (256 MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) SP(SG_CHUNK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #undef SP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static inline unsigned int sg_pool_index(unsigned short nents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) unsigned int index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) BUG_ON(nents > SG_CHUNK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) if (nents <= 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) index = get_count_order(nents) - 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) return index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static void sg_pool_free(struct scatterlist *sgl, unsigned int nents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct sg_pool *sgp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) sgp = sg_pools + sg_pool_index(nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) mempool_free(sgl, sgp->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct sg_pool *sgp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) sgp = sg_pools + sg_pool_index(nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) return mempool_alloc(sgp->pool, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * sg_free_table_chained - Free a previously mapped sg table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * @table: The sg table header to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * @nents_first_chunk: size of the first_chunk SGL passed to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * sg_alloc_table_chained
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * Free an sg table previously allocated and setup with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * sg_alloc_table_chained().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * @nents_first_chunk has to be same with that same parameter passed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * to sg_alloc_table_chained().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) void sg_free_table_chained(struct sg_table *table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) unsigned nents_first_chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (table->orig_nents <= nents_first_chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (nents_first_chunk == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) nents_first_chunk = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) EXPORT_SYMBOL_GPL(sg_free_table_chained);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * sg_alloc_table_chained - Allocate and chain SGLs in an sg table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * @table: The sg table header to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * @nents: Number of entries in sg list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * @first_chunk: first SGL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * @nents_first_chunk: number of the SGL of @first_chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * Allocate and chain SGLs in an sg table. If @nents@ is larger than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * @nents_first_chunk a chained sg table will be setup. @first_chunk is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * ignored if nents_first_chunk <= 1 because user expects the SGL points
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * non-chain SGL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) int sg_alloc_table_chained(struct sg_table *table, int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) struct scatterlist *first_chunk, unsigned nents_first_chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) BUG_ON(!nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (first_chunk && nents_first_chunk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (nents <= nents_first_chunk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) table->nents = table->orig_nents = nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) sg_init_table(table->sgl, nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) /* User supposes that the 1st SGL includes real entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) if (nents_first_chunk <= 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) first_chunk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) nents_first_chunk = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) first_chunk, nents_first_chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) GFP_ATOMIC, sg_pool_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (unlikely(ret))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) sg_free_table_chained(table, nents_first_chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) EXPORT_SYMBOL_GPL(sg_alloc_table_chained);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static __init int sg_pool_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) for (i = 0; i < SG_MEMPOOL_NR; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct sg_pool *sgp = sg_pools + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) int size = sgp->size * sizeof(struct scatterlist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) sgp->slab = kmem_cache_create(sgp->name, size, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) SLAB_HWCACHE_ALIGN, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (!sgp->slab) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) printk(KERN_ERR "SG_POOL: can't init sg slab %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) sgp->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) goto cleanup_sdb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) sgp->slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (!sgp->pool) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) sgp->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) goto cleanup_sdb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) cleanup_sdb:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) for (i = 0; i < SG_MEMPOOL_NR; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct sg_pool *sgp = sg_pools + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) mempool_destroy(sgp->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) kmem_cache_destroy(sgp->slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static __exit void sg_pool_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) for (i = 0; i < SG_MEMPOOL_NR; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct sg_pool *sgp = sg_pools + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) mempool_destroy(sgp->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) kmem_cache_destroy(sgp->slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) module_init(sg_pool_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) module_exit(sg_pool_exit);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags