^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/bio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/uio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/iocontext.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/blk-cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/sched/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/blk-crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <trace/events/block.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include "blk.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include "blk-rq-qos.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * Test patch to inline a certain number of bi_io_vec's inside the bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * itself, to shrink a bio data allocation from two mempool calls to one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define BIO_INLINE_VECS 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * if you change this list, also change bvec_alloc or things will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * break badly! cannot be bigger than what you can fit into an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * unsigned short
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define BV(x, n) { .nr_vecs = x, .name = "biovec-"#n }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) BV(1, 1), BV(4, 4), BV(16, 16), BV(64, 64), BV(128, 128), BV(BIO_MAX_PAGES, max),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #undef BV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * fs_bio_set is the bio_set containing bio and iovec memory pools used by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * IO code that does not need private memory pools.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct bio_set fs_bio_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) EXPORT_SYMBOL(fs_bio_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * Our slab pool management
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct bio_slab {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct kmem_cache *slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) unsigned int slab_ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) unsigned int slab_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) char name[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static DEFINE_MUTEX(bio_slab_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static struct bio_slab *bio_slabs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) static unsigned int bio_slab_nr, bio_slab_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) unsigned int sz = sizeof(struct bio) + extra_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct kmem_cache *slab = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct bio_slab *bslab, *new_bio_slabs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) unsigned int new_bio_slab_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) unsigned int i, entry = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) mutex_lock(&bio_slab_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) while (i < bio_slab_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) bslab = &bio_slabs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (!bslab->slab && entry == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) entry = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) else if (bslab->slab_size == sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) slab = bslab->slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) bslab->slab_ref++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if (bio_slab_nr == bio_slab_max && entry == -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) new_bio_slab_max = bio_slab_max << 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) new_bio_slabs = krealloc(bio_slabs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) new_bio_slab_max * sizeof(struct bio_slab),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) if (!new_bio_slabs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) bio_slab_max = new_bio_slab_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) bio_slabs = new_bio_slabs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) if (entry == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) entry = bio_slab_nr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) bslab = &bio_slabs[entry];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) SLAB_HWCACHE_ALIGN, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (!slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) bslab->slab = slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) bslab->slab_ref = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) bslab->slab_size = sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) mutex_unlock(&bio_slab_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) return slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) static void bio_put_slab(struct bio_set *bs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) struct bio_slab *bslab = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) mutex_lock(&bio_slab_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) for (i = 0; i < bio_slab_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (bs->bio_slab == bio_slabs[i].slab) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) bslab = &bio_slabs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) WARN_ON(!bslab->slab_ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (--bslab->slab_ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) kmem_cache_destroy(bslab->slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) bslab->slab = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) mutex_unlock(&bio_slab_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) unsigned int bvec_nr_vecs(unsigned short idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return bvec_slabs[--idx].nr_vecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) if (!idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) idx--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) BIO_BUG_ON(idx >= BVEC_POOL_NR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (idx == BVEC_POOL_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) mempool_free(bv, pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct biovec_slab *bvs = bvec_slabs + idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) kmem_cache_free(bvs->slab, bv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) mempool_t *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct bio_vec *bvl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * see comment near bvec_array define!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) switch (nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) case 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) *idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) case 2 ... 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) *idx = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) case 5 ... 16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) *idx = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) case 17 ... 64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) *idx = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) case 65 ... 128:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) *idx = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) case 129 ... BIO_MAX_PAGES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) *idx = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * idx now points to the pool we want to allocate from. only the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * 1-vec entry pool is mempool backed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (*idx == BVEC_POOL_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) fallback:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) bvl = mempool_alloc(pool, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct biovec_slab *bvs = bvec_slabs + *idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * Make this allocation restricted and don't dump info on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * allocation failures, since we'll fallback to the mempool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * in case of failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * is set, retry with the 1-entry mempool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) *idx = BVEC_POOL_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) goto fallback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) (*idx)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) return bvl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) void bio_uninit(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) #ifdef CONFIG_BLK_CGROUP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (bio->bi_blkg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) blkg_put(bio->bi_blkg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) bio->bi_blkg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (bio_integrity(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) bio_integrity_free(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) bio_crypt_free_ctx(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) EXPORT_SYMBOL(bio_uninit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) static void bio_free(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct bio_set *bs = bio->bi_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) bio_uninit(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (bs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * If we have front padding, adjust the bio pointer before freeing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) p = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) p -= bs->front_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) mempool_free(p, &bs->bio_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) /* Bio was allocated by bio_kmalloc() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) kfree(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * Users of this function have their own bio allocation. Subsequently,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * they must remember to pair any call to bio_init() with bio_uninit()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * when IO has completed, or when the bio is released.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) void bio_init(struct bio *bio, struct bio_vec *table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) unsigned short max_vecs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) memset(bio, 0, sizeof(*bio));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) atomic_set(&bio->__bi_remaining, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) atomic_set(&bio->__bi_cnt, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) bio->bi_io_vec = table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) bio->bi_max_vecs = max_vecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) EXPORT_SYMBOL(bio_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * bio_reset - reinitialize a bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * @bio: bio to reset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * After calling bio_reset(), @bio will be in the same state as a freshly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * allocated bio returned bio bio_alloc_bioset() - the only fields that are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * preserved are the ones that are initialized by bio_alloc_bioset(). See
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * comment in struct bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) void bio_reset(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) bio_uninit(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) memset(bio, 0, BIO_RESET_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) bio->bi_flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) atomic_set(&bio->__bi_remaining, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) EXPORT_SYMBOL(bio_reset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static struct bio *__bio_chain_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) struct bio *parent = bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (bio->bi_status && !parent->bi_status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) parent->bi_status = bio->bi_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) return parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) static void bio_chain_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) bio_endio(__bio_chain_endio(bio));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * bio_chain - chain bio completions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * @bio: the target bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * @parent: the parent bio of @bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * The caller won't have a bi_end_io called when @bio completes - instead,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * @parent's bi_end_io won't be called until both @parent and @bio have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * completed; the chained bio will also be freed when it completes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * The caller must not set bi_private or bi_end_io in @bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) void bio_chain(struct bio *bio, struct bio *parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) BUG_ON(bio->bi_private || bio->bi_end_io);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) bio->bi_private = parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) bio->bi_end_io = bio_chain_endio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) bio_inc_remaining(parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) EXPORT_SYMBOL(bio_chain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static void bio_alloc_rescue(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) spin_lock(&bs->rescue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) bio = bio_list_pop(&bs->rescue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) spin_unlock(&bs->rescue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (!bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static void punt_bios_to_rescuer(struct bio_set *bs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct bio_list punt, nopunt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (WARN_ON_ONCE(!bs->rescue_workqueue))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * In order to guarantee forward progress we must punt only bios that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * were allocated from this bio_set; otherwise, if there was a bio on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * there for a stacking driver higher up in the stack, processing it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * could require allocating bios from this bio_set, and doing that from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * our own rescuer would be bad.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * Since bio lists are singly linked, pop them all instead of trying to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * remove from the middle of the list:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) bio_list_init(&punt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) bio_list_init(&nopunt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) while ((bio = bio_list_pop(¤t->bio_list[0])))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) current->bio_list[0] = nopunt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) bio_list_init(&nopunt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) while ((bio = bio_list_pop(¤t->bio_list[1])))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) current->bio_list[1] = nopunt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) spin_lock(&bs->rescue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) bio_list_merge(&bs->rescue_list, &punt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) spin_unlock(&bs->rescue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) queue_work(bs->rescue_workqueue, &bs->rescue_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * bio_alloc_bioset - allocate a bio for I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * @gfp_mask: the GFP_* mask given to the slab allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * @nr_iovecs: number of iovecs to pre-allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * @bs: the bio_set to allocate from.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * backed by the @bs's mempool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * always be able to allocate a bio. This is due to the mempool guarantees.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * To make this work, callers must never allocate more than 1 bio at a time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * from this pool. Callers that need to allocate more than 1 bio must always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * submit the previously allocated bio for IO before attempting to allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * a new one. Failure to do so can cause deadlocks under memory pressure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * Note that when running under submit_bio_noacct() (i.e. any block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * driver), bios are not submitted until after you return - see the code in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * submit_bio_noacct() that converts recursion into iteration, to prevent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * stack overflows.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * This would normally mean allocating multiple bios under
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * submit_bio_noacct() would be susceptible to deadlocks, but we have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * deadlock avoidance code that resubmits any blocked bios from a rescuer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * However, we do not guarantee forward progress for allocations from other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * mempools. Doing multiple allocations from the same mempool under
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * for per bio allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * Pointer to new bio on success, NULL on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) struct bio_set *bs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) gfp_t saved_gfp = gfp_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) unsigned front_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) unsigned inline_vecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) struct bio_vec *bvl = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) if (!bs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) if (nr_iovecs > UIO_MAXIOV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) p = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) front_pad = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) inline_vecs = nr_iovecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) /* should not use nobvec bioset for nr_iovecs > 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) nr_iovecs > 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * submit_bio_noacct() converts recursion to iteration; this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * means if we're running beneath it, any bios we allocate and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * submit will not be submitted (and thus freed) until after we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * This exposes us to a potential deadlock if we allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * multiple bios from the same bio_set() while running
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * underneath submit_bio_noacct(). If we were to allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * multiple bios (say a stacking block driver that was splitting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * bios), we would deadlock if we exhausted the mempool's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * reserve.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) * We solve this, and guarantee forward progress, with a rescuer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * workqueue per bio_set. If we go to allocate and there are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * bios on current->bio_list, we first try the allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * without __GFP_DIRECT_RECLAIM; if that fails, we punt those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) * bios we would be blocking to the rescuer workqueue before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * we retry with the original gfp_flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if (current->bio_list &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) (!bio_list_empty(¤t->bio_list[0]) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) !bio_list_empty(¤t->bio_list[1])) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) bs->rescue_workqueue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) gfp_mask &= ~__GFP_DIRECT_RECLAIM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) p = mempool_alloc(&bs->bio_pool, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (!p && gfp_mask != saved_gfp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) punt_bios_to_rescuer(bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) gfp_mask = saved_gfp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) p = mempool_alloc(&bs->bio_pool, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) front_pad = bs->front_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) inline_vecs = BIO_INLINE_VECS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (unlikely(!p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) bio = p + front_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) bio_init(bio, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (nr_iovecs > inline_vecs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) unsigned long idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) if (!bvl && gfp_mask != saved_gfp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) punt_bios_to_rescuer(bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) gfp_mask = saved_gfp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (unlikely(!bvl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) goto err_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) bio->bi_flags |= idx << BVEC_POOL_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) } else if (nr_iovecs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) bvl = bio->bi_inline_vecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) bio->bi_pool = bs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) bio->bi_max_vecs = nr_iovecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) bio->bi_io_vec = bvl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) return bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) err_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) mempool_free(p, &bs->bio_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) EXPORT_SYMBOL(bio_alloc_bioset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct bio_vec bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) struct bvec_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) __bio_for_each_segment(bv, bio, iter, start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) char *data = bvec_kmap_irq(&bv, &flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) memset(data, 0, bv.bv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) flush_dcache_page(bv.bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) bvec_kunmap_irq(data, &flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) EXPORT_SYMBOL(zero_fill_bio_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * bio_truncate - truncate the bio to small size of @new_size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * @bio: the bio to be truncated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * @new_size: new size for truncating the bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * Truncate the bio to new size of @new_size. If bio_op(bio) is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * REQ_OP_READ, zero the truncated part. This function should only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * be used for handling corner cases, such as bio eod.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) void bio_truncate(struct bio *bio, unsigned new_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) struct bio_vec bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct bvec_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) unsigned int done = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) bool truncated = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (new_size >= bio->bi_iter.bi_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (bio_op(bio) != REQ_OP_READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) bio_for_each_segment(bv, bio, iter) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if (done + bv.bv_len > new_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) unsigned offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) if (!truncated)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) offset = new_size - done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) zero_user(bv.bv_page, bv.bv_offset + offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) bv.bv_len - offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) truncated = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) done += bv.bv_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * Don't touch bvec table here and make it really immutable, since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * fs bio user has to retrieve all pages via bio_for_each_segment_all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) * in its .end_bio() callback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * It is enough to truncate bio by updating .bi_size since we can make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * correct bvec with the updated .bi_size for drivers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) bio->bi_iter.bi_size = new_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) * guard_bio_eod - truncate a BIO to fit the block device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) * @bio: bio to truncate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) * This allows us to do IO even on the odd last sectors of a device, even if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) * block size is some multiple of the physical sector size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) * We'll just truncate the bio to the size of the device, and clear the end of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) * the buffer head manually. Truly out-of-range accesses will turn into actual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) * I/O errors, this only handles the "we need to be able to do I/O at the final
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) * sector" case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) void guard_bio_eod(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) sector_t maxsector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) struct hd_struct *part;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) part = __disk_get_part(bio->bi_disk, bio->bi_partno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) if (part)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) maxsector = part_nr_sects_read(part);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) maxsector = get_capacity(bio->bi_disk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) if (!maxsector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) * If the *whole* IO is past the end of the device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * let it through, and the IO layer will turn it into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * an EIO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) if (unlikely(bio->bi_iter.bi_sector >= maxsector))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) maxsector -= bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) bio_truncate(bio, maxsector << 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) * bio_put - release a reference to a bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) * @bio: bio to release reference to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * Put a reference to a &struct bio, either one you have gotten with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) void bio_put(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) if (!bio_flagged(bio, BIO_REFFED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) bio_free(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * last put frees it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (atomic_dec_and_test(&bio->__bi_cnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) bio_free(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) EXPORT_SYMBOL(bio_put);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * __bio_clone_fast - clone a bio that shares the original bio's biovec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) * @bio: destination bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * @bio_src: bio to clone
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * Clone a &bio. Caller will own the returned bio, but not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * the actual data it points to. Reference count of returned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * bio will be one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * Caller must ensure that @bio_src is not freed before @bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * most users will be overriding ->bi_disk with a new target,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) * so we don't set nor calculate new physical/hw segment counts here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) bio->bi_disk = bio_src->bi_disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) bio->bi_partno = bio_src->bi_partno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) bio_set_flag(bio, BIO_CLONED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) if (bio_flagged(bio_src, BIO_THROTTLED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) bio_set_flag(bio, BIO_THROTTLED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) bio->bi_opf = bio_src->bi_opf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) bio->bi_ioprio = bio_src->bi_ioprio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) bio->bi_write_hint = bio_src->bi_write_hint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) bio->bi_iter = bio_src->bi_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) bio->bi_io_vec = bio_src->bi_io_vec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) bio_clone_blkg_association(bio, bio_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) blkcg_bio_issue_init(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) EXPORT_SYMBOL(__bio_clone_fast);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * bio_clone_fast - clone a bio that shares the original bio's biovec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * @bio: bio to clone
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * @gfp_mask: allocation priority
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * @bs: bio_set to allocate from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * Like __bio_clone_fast, only also allocates the returned bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) struct bio *b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) b = bio_alloc_bioset(gfp_mask, 0, bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) if (!b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) __bio_clone_fast(b, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) if (bio_crypt_clone(b, bio, gfp_mask) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) goto err_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (bio_integrity(bio) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) bio_integrity_clone(b, bio, gfp_mask) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) goto err_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) return b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) err_put:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) bio_put(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) EXPORT_SYMBOL(bio_clone_fast);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) const char *bio_devname(struct bio *bio, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) return disk_name(bio->bi_disk, bio->bi_partno, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) EXPORT_SYMBOL(bio_devname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) static inline bool page_is_mergeable(const struct bio_vec *bv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) struct page *page, unsigned int len, unsigned int off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) bool *same_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) size_t bv_end = bv->bv_offset + bv->bv_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + bv_end - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) phys_addr_t page_addr = page_to_phys(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (vec_end_addr + 1 != page_addr + off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) if (xen_domain() && !xen_biovec_phys_mergeable(bv, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) *same_page = ((vec_end_addr & PAGE_MASK) == page_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) if (*same_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) return (bv->bv_page + bv_end / PAGE_SIZE) == (page + off / PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * Try to merge a page into a segment, while obeying the hardware segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * size limit. This is not for normal read/write bios, but for passthrough
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * or Zone Append operations that we can't split.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) static bool bio_try_merge_hw_seg(struct request_queue *q, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) struct page *page, unsigned len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) unsigned offset, bool *same_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) unsigned long mask = queue_segment_boundary(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) phys_addr_t addr2 = page_to_phys(page) + offset + len - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if ((addr1 | mask) != (addr2 | mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (bv->bv_len + len > queue_max_segment_size(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return __bio_try_merge_page(bio, page, len, offset, same_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * bio_add_hw_page - attempt to add a page to a bio with hw constraints
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * @q: the target queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * @bio: destination bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) * @page: page to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * @len: vec entry length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * @offset: vec entry offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) * @max_sectors: maximum number of sectors that can be added
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) * @same_page: return if the segment has been merged inside the same page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * Add a page to a bio while respecting the hardware max_sectors, max_segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * and gap limitations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) int bio_add_hw_page(struct request_queue *q, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) struct page *page, unsigned int len, unsigned int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) unsigned int max_sectors, bool *same_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) struct bio_vec *bvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) if (bio->bi_vcnt > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) if (bio_try_merge_hw_seg(q, bio, page, len, offset, same_page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) * If the queue doesn't support SG gaps and adding this segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) * would create a gap, disallow it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) if (bvec_gap_to_prev(q, bvec, offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) if (bio_full(bio, len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if (bio->bi_vcnt >= queue_max_segments(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) bvec = &bio->bi_io_vec[bio->bi_vcnt];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) bvec->bv_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) bvec->bv_len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) bvec->bv_offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) bio->bi_vcnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) bio->bi_iter.bi_size += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * bio_add_pc_page - attempt to add page to passthrough bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * @q: the target queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * @bio: destination bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) * @page: page to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * @len: vec entry length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * @offset: vec entry offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * Attempt to add a page to the bio_vec maplist. This can fail for a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * number of reasons, such as the bio being full or target block device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) * limitations. The target block device must allow bio's up to PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * so it is always possible to add a single page to an empty bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * This should only be used by passthrough bios.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) int bio_add_pc_page(struct request_queue *q, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) struct page *page, unsigned int len, unsigned int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) bool same_page = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) return bio_add_hw_page(q, bio, page, len, offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) queue_max_hw_sectors(q), &same_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) EXPORT_SYMBOL(bio_add_pc_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) * __bio_try_merge_page - try appending data to an existing bvec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) * @bio: destination bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) * @page: start page to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) * @len: length of the data to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * @off: offset of the data relative to @page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) * @same_page: return if the segment has been merged inside the same page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) * Try to add the data at @page + @off to the last bvec of @bio. This is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * useful optimisation for file systems with a block size smaller than the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * page size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) * Warn if (@len, @off) crosses pages in case that @same_page is true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * Return %true on success or %false on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) bool __bio_try_merge_page(struct bio *bio, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) unsigned int len, unsigned int off, bool *same_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (bio->bi_vcnt > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) if (page_is_mergeable(bv, page, len, off, same_page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) if (bio->bi_iter.bi_size > UINT_MAX - len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) *same_page = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) bv->bv_len += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) bio->bi_iter.bi_size += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) EXPORT_SYMBOL_GPL(__bio_try_merge_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * __bio_add_page - add page(s) to a bio in a new segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * @bio: destination bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * @page: start page to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * @len: length of the data to add, may cross pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * @off: offset of the data relative to @page, may cross pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) * Add the data at @page + @off to @bio as a new bvec. The caller must ensure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) * that @bio has space for another bvec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) void __bio_add_page(struct bio *bio, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) unsigned int len, unsigned int off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) WARN_ON_ONCE(bio_full(bio, len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) bv->bv_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) bv->bv_offset = off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) bv->bv_len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) bio->bi_iter.bi_size += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) bio->bi_vcnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) bio_set_flag(bio, BIO_WORKINGSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) EXPORT_SYMBOL_GPL(__bio_add_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) * bio_add_page - attempt to add page(s) to bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) * @bio: destination bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) * @page: start page to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * @len: vec entry length, may cross pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * @offset: vec entry offset relative to @page, may cross pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) * Attempt to add page(s) to the bio_vec maplist. This will only fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) int bio_add_page(struct bio *bio, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) unsigned int len, unsigned int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) bool same_page = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) if (bio_full(bio, len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) __bio_add_page(bio, page, len, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) EXPORT_SYMBOL(bio_add_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) void bio_release_pages(struct bio *bio, bool mark_dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) struct bvec_iter_all iter_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) struct bio_vec *bvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) if (bio_flagged(bio, BIO_NO_PAGE_REF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) bio_for_each_segment_all(bvec, bio, iter_all) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) if (mark_dirty && !PageCompound(bvec->bv_page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) set_page_dirty_lock(bvec->bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) put_page(bvec->bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) EXPORT_SYMBOL_GPL(bio_release_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) const struct bio_vec *bv = iter->bvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) unsigned int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) if (WARN_ON_ONCE(iter->iov_offset > bv->bv_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) size = bio_add_page(bio, bv->bv_page, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) bv->bv_offset + iter->iov_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) if (unlikely(size != len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) iov_iter_advance(iter, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) static void bio_put_pages(struct page **pages, size_t size, size_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) size_t i, nr = DIV_ROUND_UP(size + (off & ~PAGE_MASK), PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) for (i = 0; i < nr; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) put_page(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) #define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) * @bio: bio to add pages to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) * @iter: iov iterator describing the region to be mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) * Pins pages from *iter and appends them to @bio's bvec array. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) * pages will have to be released using put_page() when done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) * For multi-segment *iter, this function only adds pages from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) * next non-empty segment of the iov iterator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) struct page **pages = (struct page **)bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) bool same_page = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) ssize_t size, left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) unsigned len, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) size_t offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * Move page array up in the allocated memory for the bio vecs as far as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) * possible so that we can start filling biovecs from the beginning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) * without overwriting the temporary page array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) if (unlikely(size <= 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) return size ? size : -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) for (left = size, i = 0; left > 0; left -= len, i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) struct page *page = pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) len = min_t(size_t, PAGE_SIZE - offset, left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) if (__bio_try_merge_page(bio, page, len, offset, &same_page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if (same_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) if (WARN_ON_ONCE(bio_full(bio, len))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) bio_put_pages(pages + i, left, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) __bio_add_page(bio, page, len, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) iov_iter_advance(iter, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) struct request_queue *q = bio->bi_disk->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) unsigned int max_append_sectors = queue_max_zone_append_sectors(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) struct page **pages = (struct page **)bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) ssize_t size, left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) unsigned len, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) size_t offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) if (WARN_ON_ONCE(!max_append_sectors))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) * Move page array up in the allocated memory for the bio vecs as far as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) * possible so that we can start filling biovecs from the beginning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) * without overwriting the temporary page array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) if (unlikely(size <= 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) return size ? size : -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) for (left = size, i = 0; left > 0; left -= len, i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) struct page *page = pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) bool same_page = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) len = min_t(size_t, PAGE_SIZE - offset, left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if (bio_add_hw_page(q, bio, page, len, offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) max_append_sectors, &same_page) != len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) bio_put_pages(pages + i, left, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) if (same_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) iov_iter_advance(iter, size - left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) * bio_iov_iter_get_pages - add user or kernel pages to a bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) * @bio: bio to add pages to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * @iter: iov iterator describing the region to be added
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * This takes either an iterator pointing to user memory, or one pointing to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) * map them into the kernel. On IO completion, the caller should put those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) * pages. If we're adding kernel pages, and the caller told us it's safe to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) * do so, we just have to add the pages to the bio directly. We don't grab an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) * extra reference to those pages (the user should already have that), and we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) * don't put the page on IO completion. The caller needs to check if the bio is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) * flagged BIO_NO_PAGE_REF on IO completion. If it isn't, then pages should be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) * released.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) * The function tries, but does not guarantee, to pin as many pages as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) * fit into the bio, or are requested in @iter, whatever is smaller. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) * MM encounters an error pinning the requested pages, it stops. Error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) * is returned only if 0 pages could be pinned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) const bool is_bvec = iov_iter_is_bvec(iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) if (WARN_ON_ONCE(bio->bi_vcnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) if (WARN_ON_ONCE(is_bvec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) ret = __bio_iov_append_get_pages(bio, iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) if (is_bvec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) ret = __bio_iov_bvec_add_pages(bio, iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) ret = __bio_iov_iter_get_pages(bio, iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) if (is_bvec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) bio_set_flag(bio, BIO_NO_PAGE_REF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) return bio->bi_vcnt ? 0 : ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) static void submit_bio_wait_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) complete(bio->bi_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) * submit_bio_wait - submit a bio, and wait until it completes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) * @bio: The &struct bio which describes the I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) * bio_endio() on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) * WARNING: Unlike to how submit_bio() is usually used, this function does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) * result in bio reference to be consumed. The caller must drop the reference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) * on his own.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) int submit_bio_wait(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_disk->lockdep_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) unsigned long hang_check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) bio->bi_private = &done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) bio->bi_end_io = submit_bio_wait_endio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) bio->bi_opf |= REQ_SYNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) submit_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) /* Prevent hang_check timer from firing at us during very long I/O */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) hang_check = sysctl_hung_task_timeout_secs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) if (hang_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) while (!wait_for_completion_io_timeout(&done,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) hang_check * (HZ/2)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) wait_for_completion_io(&done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) return blk_status_to_errno(bio->bi_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) EXPORT_SYMBOL(submit_bio_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) * bio_advance - increment/complete a bio by some number of bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) * @bio: bio to advance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) * @bytes: number of bytes to complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) * be updated on the last bvec as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) * @bio will then represent the remaining, uncompleted portion of the io.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) void bio_advance(struct bio *bio, unsigned bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (bio_integrity(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) bio_integrity_advance(bio, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) bio_crypt_advance(bio, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) bio_advance_iter(bio, &bio->bi_iter, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) EXPORT_SYMBOL(bio_advance);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) struct bio *src, struct bvec_iter *src_iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) struct bio_vec src_bv, dst_bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) void *src_p, *dst_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) unsigned bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) while (src_iter->bi_size && dst_iter->bi_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) src_bv = bio_iter_iovec(src, *src_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) dst_bv = bio_iter_iovec(dst, *dst_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) bytes = min(src_bv.bv_len, dst_bv.bv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) src_p = kmap_atomic(src_bv.bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) dst_p = kmap_atomic(dst_bv.bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) memcpy(dst_p + dst_bv.bv_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) src_p + src_bv.bv_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) kunmap_atomic(dst_p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) kunmap_atomic(src_p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) flush_dcache_page(dst_bv.bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) bio_advance_iter(src, src_iter, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) bio_advance_iter(dst, dst_iter, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) EXPORT_SYMBOL(bio_copy_data_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) * bio_copy_data - copy contents of data buffers from one bio to another
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) * @src: source bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * @dst: destination bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * Stops when it reaches the end of either @src or @dst - that is, copies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) void bio_copy_data(struct bio *dst, struct bio *src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) struct bvec_iter src_iter = src->bi_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) struct bvec_iter dst_iter = dst->bi_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) EXPORT_SYMBOL(bio_copy_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) * bio_list_copy_data - copy contents of data buffers from one chain of bios to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * another
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) * @src: source bio list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) * @dst: destination bio list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) * Stops when it reaches the end of either the @src list or @dst list - that is,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) * copies min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) * bios).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) void bio_list_copy_data(struct bio *dst, struct bio *src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) struct bvec_iter src_iter = src->bi_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) struct bvec_iter dst_iter = dst->bi_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) if (!src_iter.bi_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) src = src->bi_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) if (!src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) src_iter = src->bi_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) if (!dst_iter.bi_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) dst = dst->bi_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if (!dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) dst_iter = dst->bi_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) EXPORT_SYMBOL(bio_list_copy_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) void bio_free_pages(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) struct bio_vec *bvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) struct bvec_iter_all iter_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) bio_for_each_segment_all(bvec, bio, iter_all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) __free_page(bvec->bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) EXPORT_SYMBOL(bio_free_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) * for performing direct-IO in BIOs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) * The problem is that we cannot run set_page_dirty() from interrupt context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) * because the required locks are not interrupt-safe. So what we can do is to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) * mark the pages dirty _before_ performing IO. And in interrupt context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) * check that the pages are still dirty. If so, fine. If not, redirty them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) * in process context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) * We special-case compound pages here: normally this means reads into hugetlb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) * pages. The logic in here doesn't really work right for compound pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) * because the VM does not uniformly chase down the head page in all cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) * handle them at all. So we skip compound pages here at an early stage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) * Note that this code is very hard to test under normal circumstances because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) * direct-io pins the pages with get_user_pages(). This makes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) * is_page_cache_freeable return false, and the VM will not clean the pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) * But other code (eg, flusher threads) could clean the pages if they are mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) * pagecache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) * deferred bio dirtying paths.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) * bio_set_pages_dirty() will mark all the bio's pages as dirty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) void bio_set_pages_dirty(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) struct bio_vec *bvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) struct bvec_iter_all iter_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) bio_for_each_segment_all(bvec, bio, iter_all) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) if (!PageCompound(bvec->bv_page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) set_page_dirty_lock(bvec->bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) * If they are, then fine. If, however, some pages are clean then they must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) * have been written out during the direct-IO read. So we take another ref on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) * the BIO and re-dirty the pages in process context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) * It is expected that bio_check_pages_dirty() will wholly own the BIO from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) * here on. It will run one put_page() against each page and will run one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) * bio_put() against the BIO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) static void bio_dirty_fn(struct work_struct *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) static DEFINE_SPINLOCK(bio_dirty_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) static struct bio *bio_dirty_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) * This runs in process context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) static void bio_dirty_fn(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) struct bio *bio, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) spin_lock_irq(&bio_dirty_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) next = bio_dirty_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) bio_dirty_list = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) spin_unlock_irq(&bio_dirty_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) while ((bio = next) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) next = bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) bio_release_pages(bio, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) void bio_check_pages_dirty(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) struct bio_vec *bvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) struct bvec_iter_all iter_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) bio_for_each_segment_all(bvec, bio, iter_all) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) goto defer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) bio_release_pages(bio, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) defer:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) spin_lock_irqsave(&bio_dirty_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) bio->bi_private = bio_dirty_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) bio_dirty_list = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) spin_unlock_irqrestore(&bio_dirty_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) schedule_work(&bio_dirty_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) static inline bool bio_remaining_done(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) * If we're not chaining, then ->__bi_remaining is always 1 and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) * we always end io on the first invocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) if (!bio_flagged(bio, BIO_CHAIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (atomic_dec_and_test(&bio->__bi_remaining)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) bio_clear_flag(bio, BIO_CHAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) * bio_endio - end I/O on a bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) * @bio: bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) * way to end I/O on a bio. No one should call bi_end_io() directly on a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) * bio unless they own it and thus know that it has an end_io function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) * bio_endio() can be called several times on a bio that has been chained
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) * using bio_chain(). The ->bi_end_io() function will only be called the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) * last time. At this point the BLK_TA_COMPLETE tracing event will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) * generated if BIO_TRACE_COMPLETION is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) void bio_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) if (!bio_remaining_done(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) if (!bio_integrity_endio(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) if (bio->bi_disk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) rq_qos_done_bio(bio->bi_disk->queue, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) * Need to have a real endio function for chained bios, otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) * various corner cases will break (like stacking block devices that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) * save/restore bi_end_io) - however, we want to avoid unbounded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) * recursion and blowing the stack. Tail call optimization would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) * handle this, but compiling with frame pointers also disables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) * gcc's sibling call optimization.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) if (bio->bi_end_io == bio_chain_endio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) bio = __bio_chain_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) if (bio->bi_disk && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) trace_block_bio_complete(bio->bi_disk->queue, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) bio_clear_flag(bio, BIO_TRACE_COMPLETION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) blk_throtl_bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) /* release cgroup info */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) bio_uninit(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) if (bio->bi_end_io)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) bio->bi_end_io(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) EXPORT_SYMBOL(bio_endio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) * bio_split - split a bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) * @bio: bio to split
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) * @sectors: number of sectors to split from the front of @bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) * @gfp: gfp mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) * @bs: bio set to allocate from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) * Allocates and returns a new bio which represents @sectors from the start of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) * @bio, and updates @bio to represent the remaining sectors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) * Unless this is a discard request the newly allocated bio will point
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) * to @bio's bi_io_vec. It is the caller's responsibility to ensure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) * neither @bio nor @bs are freed before the split bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) struct bio *bio_split(struct bio *bio, int sectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) gfp_t gfp, struct bio_set *bs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) struct bio *split;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) BUG_ON(sectors <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) BUG_ON(sectors >= bio_sectors(bio));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) /* Zone append commands cannot be split */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) split = bio_clone_fast(bio, gfp, bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) if (!split)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) split->bi_iter.bi_size = sectors << 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) if (bio_integrity(split))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) bio_integrity_trim(split);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) bio_advance(bio, split->bi_iter.bi_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) if (bio_flagged(bio, BIO_TRACE_COMPLETION))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) bio_set_flag(split, BIO_TRACE_COMPLETION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) return split;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) EXPORT_SYMBOL(bio_split);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) * bio_trim - trim a bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) * @bio: bio to trim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) * @offset: number of sectors to trim from the front of @bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) * @size: size we want to trim @bio to, in sectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) void bio_trim(struct bio *bio, int offset, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) /* 'bio' is a cloned bio which we need to trim to match
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) * the given offset and size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) size <<= 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) if (offset == 0 && size == bio->bi_iter.bi_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) bio_advance(bio, offset << 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) bio->bi_iter.bi_size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) if (bio_integrity(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) bio_integrity_trim(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) EXPORT_SYMBOL_GPL(bio_trim);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) * create memory pools for biovec's in a bio_set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) * use the global biovec slabs created for general use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) int biovec_init_pool(mempool_t *pool, int pool_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) return mempool_init_slab_pool(pool, pool_entries, bp->slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) * bioset_exit - exit a bioset initialized with bioset_init()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) * May be called on a zeroed but uninitialized bioset (i.e. allocated with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) * kzalloc()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) void bioset_exit(struct bio_set *bs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) if (bs->rescue_workqueue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) destroy_workqueue(bs->rescue_workqueue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) bs->rescue_workqueue = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) mempool_exit(&bs->bio_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) mempool_exit(&bs->bvec_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) bioset_integrity_free(bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) if (bs->bio_slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) bio_put_slab(bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) bs->bio_slab = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) EXPORT_SYMBOL(bioset_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) * bioset_init - Initialize a bio_set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) * @bs: pool to initialize
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) * @pool_size: Number of bio and bio_vecs to cache in the mempool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) * @front_pad: Number of bytes to allocate in front of the returned bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) * @flags: Flags to modify behavior, currently %BIOSET_NEED_BVECS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) * and %BIOSET_NEED_RESCUER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) * to ask for a number of bytes to be allocated in front of the bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) * Front pad allocation is useful for embedding the bio inside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) * another structure, to avoid allocating extra data to go with the bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) * Note that the bio must be embedded at the END of that structure always,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) * or things will break badly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) * If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) * for allocating iovecs. This pool is not needed e.g. for bio_clone_fast().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) * If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) * dispatch queued requests when the mempool runs out of space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) int bioset_init(struct bio_set *bs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) unsigned int pool_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) unsigned int front_pad,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) bs->front_pad = front_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) spin_lock_init(&bs->rescue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) bio_list_init(&bs->rescue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) if (!bs->bio_slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) if ((flags & BIOSET_NEED_BVECS) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) biovec_init_pool(&bs->bvec_pool, pool_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) if (!(flags & BIOSET_NEED_RESCUER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) if (!bs->rescue_workqueue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) bad:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) bioset_exit(bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) EXPORT_SYMBOL(bioset_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) * Initialize and setup a new bio_set, based on the settings from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) * another bio_set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) int flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) if (src->bvec_pool.min_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) flags |= BIOSET_NEED_BVECS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) if (src->rescue_workqueue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) flags |= BIOSET_NEED_RESCUER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) EXPORT_SYMBOL(bioset_init_from_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) static void __init biovec_init_slabs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) for (i = 0; i < BVEC_POOL_NR; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) struct biovec_slab *bvs = bvec_slabs + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) if (bvs->nr_vecs <= BIO_INLINE_VECS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) bvs->slab = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) size = bvs->nr_vecs * sizeof(struct bio_vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) bvs->slab = kmem_cache_create(bvs->name, size, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) static int __init init_bio(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) bio_slab_max = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) bio_slab_nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) bio_slabs = kcalloc(bio_slab_max, sizeof(struct bio_slab),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) if (!bio_slabs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) panic("bio: can't allocate bios\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) bio_integrity_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) biovec_init_slabs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) panic("bio: can't allocate bios\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) panic("bio: can't create integrity pool\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) subsys_initcall(init_bio);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags