^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright (C) 2012-2017 Red Hat, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is released under the GPL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include "dm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "dm-bio-prison-v2.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/rwsem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define MIN_CELLS 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) struct dm_bio_prison_v2 {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) struct workqueue_struct *wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) struct rb_root cells;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) mempool_t cell_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static struct kmem_cache *_cell_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * @nr_cells should be the number of cells you want in use _concurrently_.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * Don't confuse it with the number of distinct keys.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) struct dm_bio_prison_v2 *dm_bio_prison_create_v2(struct workqueue_struct *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct dm_bio_prison_v2 *prison = kzalloc(sizeof(*prison), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) if (!prison)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) prison->wq = wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) spin_lock_init(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) kfree(prison);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) prison->cells = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) return prison;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) EXPORT_SYMBOL_GPL(dm_bio_prison_create_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) void dm_bio_prison_destroy_v2(struct dm_bio_prison_v2 *prison)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) mempool_exit(&prison->cell_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) kfree(prison);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) EXPORT_SYMBOL_GPL(dm_bio_prison_destroy_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct dm_bio_prison_cell_v2 *dm_bio_prison_alloc_cell_v2(struct dm_bio_prison_v2 *prison, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) return mempool_alloc(&prison->cell_pool, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) void dm_bio_prison_free_cell_v2(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct dm_bio_prison_cell_v2 *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) mempool_free(cell, &prison->cell_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static void __setup_new_cell(struct dm_cell_key_v2 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct dm_bio_prison_cell_v2 *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) memset(cell, 0, sizeof(*cell));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) memcpy(&cell->key, key, sizeof(cell->key));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) bio_list_init(&cell->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static int cmp_keys(struct dm_cell_key_v2 *lhs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct dm_cell_key_v2 *rhs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (lhs->virtual < rhs->virtual)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (lhs->virtual > rhs->virtual)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) if (lhs->dev < rhs->dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) if (lhs->dev > rhs->dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (lhs->block_end <= rhs->block_begin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) if (lhs->block_begin >= rhs->block_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * Returns true if node found, otherwise it inserts a new one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static bool __find_or_insert(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct dm_cell_key_v2 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct dm_bio_prison_cell_v2 *cell_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct dm_bio_prison_cell_v2 **result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) struct rb_node **new = &prison->cells.rb_node, *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) while (*new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct dm_bio_prison_cell_v2 *cell =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) rb_entry(*new, struct dm_bio_prison_cell_v2, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) r = cmp_keys(key, &cell->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) parent = *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) new = &((*new)->rb_left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) else if (r > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) new = &((*new)->rb_right);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) *result = cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) __setup_new_cell(key, cell_prealloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) *result = cell_prealloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) rb_link_node(&cell_prealloc->node, parent, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) rb_insert_color(&cell_prealloc->node, &prison->cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) static bool __get(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) struct dm_cell_key_v2 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) unsigned lock_level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) struct bio *inmate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct dm_bio_prison_cell_v2 *cell_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) struct dm_bio_prison_cell_v2 **cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) if (__find_or_insert(prison, key, cell_prealloc, cell)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if ((*cell)->exclusive_lock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) if (lock_level <= (*cell)->exclusive_level) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) bio_list_add(&(*cell)->bios, inmate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) (*cell)->shared_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) (*cell)->shared_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) bool dm_cell_get_v2(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) struct dm_cell_key_v2 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) unsigned lock_level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct bio *inmate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) struct dm_bio_prison_cell_v2 *cell_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) struct dm_bio_prison_cell_v2 **cell_result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) spin_lock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) r = __get(prison, key, lock_level, inmate, cell_prealloc, cell_result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) spin_unlock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) EXPORT_SYMBOL_GPL(dm_cell_get_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) static bool __put(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) struct dm_bio_prison_cell_v2 *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) BUG_ON(!cell->shared_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) cell->shared_count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) // FIXME: shared locks granted above the lock level could starve this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (!cell->shared_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (cell->exclusive_lock){
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (cell->quiesce_continuation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) queue_work(prison->wq, cell->quiesce_continuation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) cell->quiesce_continuation = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) rb_erase(&cell->node, &prison->cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) bool dm_cell_put_v2(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) struct dm_bio_prison_cell_v2 *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) bool r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) spin_lock_irqsave(&prison->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) r = __put(prison, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) spin_unlock_irqrestore(&prison->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) EXPORT_SYMBOL_GPL(dm_cell_put_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) static int __lock(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) struct dm_cell_key_v2 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) unsigned lock_level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) struct dm_bio_prison_cell_v2 *cell_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) struct dm_bio_prison_cell_v2 **cell_result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) struct dm_bio_prison_cell_v2 *cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) if (__find_or_insert(prison, key, cell_prealloc, &cell)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (cell->exclusive_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) cell->exclusive_lock = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) cell->exclusive_level = lock_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) *cell_result = cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) // FIXME: we don't yet know what level these shared locks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) // were taken at, so have to quiesce them all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) return cell->shared_count > 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) cell = cell_prealloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) cell->shared_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) cell->exclusive_lock = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) cell->exclusive_level = lock_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) *cell_result = cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) int dm_cell_lock_v2(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct dm_cell_key_v2 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) unsigned lock_level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct dm_bio_prison_cell_v2 *cell_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct dm_bio_prison_cell_v2 **cell_result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) spin_lock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) r = __lock(prison, key, lock_level, cell_prealloc, cell_result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) spin_unlock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) EXPORT_SYMBOL_GPL(dm_cell_lock_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) static void __quiesce(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) struct dm_bio_prison_cell_v2 *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct work_struct *continuation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if (!cell->shared_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) queue_work(prison->wq, continuation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) cell->quiesce_continuation = continuation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) void dm_cell_quiesce_v2(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct dm_bio_prison_cell_v2 *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) struct work_struct *continuation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) spin_lock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) __quiesce(prison, cell, continuation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) spin_unlock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) EXPORT_SYMBOL_GPL(dm_cell_quiesce_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) static int __promote(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) struct dm_bio_prison_cell_v2 *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned new_lock_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (!cell->exclusive_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) cell->exclusive_level = new_lock_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) return cell->shared_count > 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) int dm_cell_lock_promote_v2(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) struct dm_bio_prison_cell_v2 *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) unsigned new_lock_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) spin_lock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) r = __promote(prison, cell, new_lock_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) spin_unlock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) EXPORT_SYMBOL_GPL(dm_cell_lock_promote_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static bool __unlock(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct dm_bio_prison_cell_v2 *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) struct bio_list *bios)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) BUG_ON(!cell->exclusive_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) bio_list_merge(bios, &cell->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) bio_list_init(&cell->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (cell->shared_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) cell->exclusive_lock = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) rb_erase(&cell->node, &prison->cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) bool dm_cell_unlock_v2(struct dm_bio_prison_v2 *prison,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) struct dm_bio_prison_cell_v2 *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) struct bio_list *bios)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) bool r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) spin_lock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) r = __unlock(prison, cell, bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) spin_unlock_irq(&prison->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) EXPORT_SYMBOL_GPL(dm_cell_unlock_v2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) int __init dm_bio_prison_init_v2(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) _cell_cache = KMEM_CACHE(dm_bio_prison_cell_v2, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (!_cell_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) void dm_bio_prison_exit_v2(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) kmem_cache_destroy(_cell_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) _cell_cache = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags