^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright (C) 2012 Red Hat, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is released under the GPL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include "dm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "dm-bio-prison-v1.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include "dm-bio-prison-v2.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define MIN_CELLS 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) struct dm_bio_prison {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) struct rb_root cells;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) mempool_t cell_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static struct kmem_cache *_cell_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * @nr_cells should be the number of cells you want in use _concurrently_.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * Don't confuse it with the number of distinct keys.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) struct dm_bio_prison *dm_bio_prison_create(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) struct dm_bio_prison *prison = kzalloc(sizeof(*prison), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (!prison)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) spin_lock_init(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) kfree(prison);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) prison->cells = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return prison;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) EXPORT_SYMBOL_GPL(dm_bio_prison_create);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) void dm_bio_prison_destroy(struct dm_bio_prison *prison)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) mempool_exit(&prison->cell_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) kfree(prison);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) return mempool_alloc(&prison->cell_pool, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) mempool_free(cell, &prison->cell_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static void __setup_new_cell(struct dm_cell_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) struct bio *holder,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) memcpy(&cell->key, key, sizeof(cell->key));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) cell->holder = holder;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) bio_list_init(&cell->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static int cmp_keys(struct dm_cell_key *lhs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) struct dm_cell_key *rhs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (lhs->virtual < rhs->virtual)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if (lhs->virtual > rhs->virtual)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) if (lhs->dev < rhs->dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) if (lhs->dev > rhs->dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (lhs->block_end <= rhs->block_begin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (lhs->block_begin >= rhs->block_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static int __bio_detain(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) struct dm_cell_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct bio *inmate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) struct dm_bio_prison_cell *cell_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) struct dm_bio_prison_cell **cell_result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct rb_node **new = &prison->cells.rb_node, *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) while (*new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) struct dm_bio_prison_cell *cell =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) rb_entry(*new, struct dm_bio_prison_cell, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) r = cmp_keys(key, &cell->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) parent = *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) new = &((*new)->rb_left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) else if (r > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) new = &((*new)->rb_right);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) if (inmate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) bio_list_add(&cell->bios, inmate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) *cell_result = cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) __setup_new_cell(key, inmate, cell_prealloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) *cell_result = cell_prealloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) rb_link_node(&cell_prealloc->node, parent, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) rb_insert_color(&cell_prealloc->node, &prison->cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static int bio_detain(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct dm_cell_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct bio *inmate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) struct dm_bio_prison_cell *cell_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) struct dm_bio_prison_cell **cell_result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) spin_lock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) spin_unlock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) int dm_bio_detain(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct dm_cell_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct bio *inmate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) struct dm_bio_prison_cell *cell_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct dm_bio_prison_cell **cell_result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) EXPORT_SYMBOL_GPL(dm_bio_detain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) int dm_get_cell(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) struct dm_cell_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) struct dm_bio_prison_cell *cell_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct dm_bio_prison_cell **cell_result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) EXPORT_SYMBOL_GPL(dm_get_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * @inmates must have been initialised prior to this call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static void __cell_release(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct dm_bio_prison_cell *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct bio_list *inmates)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) rb_erase(&cell->node, &prison->cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (inmates) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) if (cell->holder)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) bio_list_add(inmates, cell->holder);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) bio_list_merge(inmates, &cell->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) void dm_cell_release(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) struct dm_bio_prison_cell *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct bio_list *bios)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) spin_lock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) __cell_release(prison, cell, bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) spin_unlock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) EXPORT_SYMBOL_GPL(dm_cell_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * Sometimes we don't want the holder, just the additional bios.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) static void __cell_release_no_holder(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct dm_bio_prison_cell *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) struct bio_list *inmates)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) rb_erase(&cell->node, &prison->cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) bio_list_merge(inmates, &cell->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) void dm_cell_release_no_holder(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct dm_bio_prison_cell *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) struct bio_list *inmates)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) spin_lock_irqsave(&prison->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) __cell_release_no_holder(prison, cell, inmates);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) spin_unlock_irqrestore(&prison->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) void dm_cell_error(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct dm_bio_prison_cell *cell, blk_status_t error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) dm_cell_release(prison, cell, &bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) while ((bio = bio_list_pop(&bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) bio->bi_status = error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) EXPORT_SYMBOL_GPL(dm_cell_error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) void dm_cell_visit_release(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) void (*visit_fn)(void *, struct dm_bio_prison_cell *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) void *context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) spin_lock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) visit_fn(context, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) rb_erase(&cell->node, &prison->cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) spin_unlock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) EXPORT_SYMBOL_GPL(dm_cell_visit_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) static int __promote_or_release(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (bio_list_empty(&cell->bios)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) rb_erase(&cell->node, &prison->cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) cell->holder = bio_list_pop(&cell->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) int dm_cell_promote_or_release(struct dm_bio_prison *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) spin_lock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) r = __promote_or_release(prison, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) spin_unlock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) EXPORT_SYMBOL_GPL(dm_cell_promote_or_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) #define DEFERRED_SET_SIZE 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) struct dm_deferred_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) struct dm_deferred_set *ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) unsigned count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) struct list_head work_items;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) struct dm_deferred_set {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned current_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) unsigned sweeper;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) struct dm_deferred_set *dm_deferred_set_create(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct dm_deferred_set *ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) ds = kmalloc(sizeof(*ds), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (!ds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) spin_lock_init(&ds->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) ds->current_entry = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) ds->sweeper = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) for (i = 0; i < DEFERRED_SET_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) ds->entries[i].ds = ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) ds->entries[i].count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) INIT_LIST_HEAD(&ds->entries[i].work_items);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) EXPORT_SYMBOL_GPL(dm_deferred_set_create);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) void dm_deferred_set_destroy(struct dm_deferred_set *ds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) kfree(ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) struct dm_deferred_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) spin_lock_irqsave(&ds->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) entry = ds->entries + ds->current_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) entry->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) spin_unlock_irqrestore(&ds->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) static unsigned ds_next(unsigned index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return (index + 1) % DEFERRED_SET_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) static void __sweep(struct dm_deferred_set *ds, struct list_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) while ((ds->sweeper != ds->current_entry) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) !ds->entries[ds->sweeper].count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) list_splice_init(&ds->entries[ds->sweeper].work_items, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) ds->sweeper = ds_next(ds->sweeper);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) list_splice_init(&ds->entries[ds->sweeper].work_items, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) spin_lock_irqsave(&entry->ds->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) BUG_ON(!entry->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) --entry->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) __sweep(entry->ds, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) spin_unlock_irqrestore(&entry->ds->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * Returns 1 if deferred or 0 if no pending items to delay job.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) int r = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) unsigned next_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) spin_lock_irq(&ds->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if ((ds->sweeper == ds->current_entry) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) !ds->entries[ds->current_entry].count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) list_add(work, &ds->entries[ds->current_entry].work_items);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) next_entry = ds_next(ds->current_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (!ds->entries[next_entry].count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) ds->current_entry = next_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) spin_unlock_irq(&ds->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) static int __init dm_bio_prison_init_v1(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) _cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (!_cell_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) static void dm_bio_prison_exit_v1(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) kmem_cache_destroy(_cell_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) _cell_cache = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) static int (*_inits[])(void) __initdata = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) dm_bio_prison_init_v1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) dm_bio_prison_init_v2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) static void (*_exits[])(void) = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) dm_bio_prison_exit_v1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) dm_bio_prison_exit_v2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) static int __init dm_bio_prison_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) const int count = ARRAY_SIZE(_inits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) int r, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) for (i = 0; i < count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) r = _inits[i]();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) bad:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) while (i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) _exits[i]();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) static void __exit dm_bio_prison_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) int i = ARRAY_SIZE(_exits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) while (i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) _exits[i]();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * module hooks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) module_init(dm_bio_prison_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) module_exit(dm_bio_prison_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) MODULE_DESCRIPTION(DM_NAME " bio prison");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) MODULE_LICENSE("GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags