^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright (C) 2011-2012 Red Hat UK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is released under the GPL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include "dm-thin-metadata.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "dm-bio-prison-v1.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include "dm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/device-mapper.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/dm-io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/dm-kcopyd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/log2.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/rculist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/sort.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/rbtree.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define DM_MSG_PREFIX "thin"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * Tunable constants
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define ENDIO_HOOK_POOL_SIZE 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define MAPPING_POOL_SIZE 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define COMMIT_PERIOD HZ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define NO_SPACE_TIMEOUT_SECS 60
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static unsigned no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) "A percentage of time allocated for copy on write");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * The block size of the device holding pool data must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * between 64KB and 1GB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * Device id is restricted to 24 bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define MAX_DEV_ID ((1 << 24) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * How do we handle breaking sharing of data blocks?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * =================================================
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * We use a standard copy-on-write btree to store the mappings for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * devices (note I'm talking about copy-on-write of the metadata here, not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * the data). When you take an internal snapshot you clone the root node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * of the origin btree. After this there is no concept of an origin or a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * snapshot. They are just two device trees that happen to point to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * same data blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * When we get a write in we decide if it's to a shared data block using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * some timestamp magic. If it is, we have to break sharing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * Let's say we write to a shared block in what was the origin. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * steps are:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * i) plug io further to this physical block. (see bio_prison code).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * ii) quiesce any read io to that shared data block. Obviously
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * including all devices that share this block. (see dm_deferred_set code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * iii) copy the data block to a newly allocate block. This step can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * missed out if the io covers the block. (schedule_copy).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * iv) insert the new mapping into the origin's btree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * (process_prepared_mapping). This act of inserting breaks some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * sharing of btree nodes between the two devices. Breaking sharing only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * effects the btree of that specific device. Btrees for the other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * devices that share the block never change. The btree for the origin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * device as it was after the last commit is untouched, ie. we're using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * persistent data structures in the functional programming sense.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * v) unplug io to this physical block, including the io that triggered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * the breaking of sharing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * Steps (ii) and (iii) occur in parallel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * The metadata _doesn't_ need to be committed before the io continues. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * get away with this because the io is always written to a _new_ block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * If there's a crash, then:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * - The origin mapping will point to the old origin block (the shared
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * one). This will contain the data as it was before the io that triggered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * the breaking of sharing came in.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * - The snap mapping still points to the old block. As it would after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * the commit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * The downside of this scheme is the timestamp magic isn't perfect, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * will continue to think that data block in the snapshot device is shared
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * even after the write to the origin has broken sharing. I suspect data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * blocks will typically be shared by many different devices, so we're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * breaking sharing n + 1 times, rather than n, where n is the number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * devices that reference this data block. At the moment I think the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * benefits far, far outweigh the disadvantages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * Key building.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) enum lock_space {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) VIRTUAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) PHYSICAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) static void build_key(struct dm_thin_device *td, enum lock_space ls,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) dm_block_t b, dm_block_t e, struct dm_cell_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) key->virtual = (ls == VIRTUAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) key->dev = dm_thin_dev_id(td);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) key->block_begin = b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) key->block_end = e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static void build_data_key(struct dm_thin_device *td, dm_block_t b,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) struct dm_cell_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) build_key(td, PHYSICAL, b, b + 1llu, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct dm_cell_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) build_key(td, VIRTUAL, b, b + 1llu, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define THROTTLE_THRESHOLD (1 * HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct throttle {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct rw_semaphore lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) unsigned long threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) bool throttle_applied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static void throttle_init(struct throttle *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) init_rwsem(&t->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) t->throttle_applied = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static void throttle_work_start(struct throttle *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) t->threshold = jiffies + THROTTLE_THRESHOLD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static void throttle_work_update(struct throttle *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (!t->throttle_applied && jiffies > t->threshold) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) down_write(&t->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) t->throttle_applied = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) static void throttle_work_complete(struct throttle *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (t->throttle_applied) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) t->throttle_applied = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) up_write(&t->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) static void throttle_lock(struct throttle *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) down_read(&t->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static void throttle_unlock(struct throttle *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) up_read(&t->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * A pool device ties together a metadata device and a data device. It
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * also provides the interface for creating and destroying internal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * devices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) struct dm_thin_new_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * The pool runs in various modes. Ordered in degraded order for comparisons.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) enum pool_mode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) PM_WRITE, /* metadata may be changed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) PM_OUT_OF_DATA_SPACE, /* metadata may be changed, though data may not be allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * Like READ_ONLY, except may switch back to WRITE on metadata resize. Reported as READ_ONLY.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) PM_OUT_OF_METADATA_SPACE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) PM_READ_ONLY, /* metadata may not be changed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) PM_FAIL, /* all I/O fails */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct pool_features {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) enum pool_mode mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) bool zero_new_blocks:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) bool discard_enabled:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) bool discard_passdown:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) bool error_if_no_space:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) struct thin_c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) #define CELL_SORT_ARRAY_SIZE 8192
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) struct pool {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) struct dm_target *ti; /* Only set if a pool target is bound */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct mapped_device *pool_md;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct block_device *data_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct block_device *md_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) struct dm_pool_metadata *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) dm_block_t low_water_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) uint32_t sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) int sectors_per_block_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) struct pool_features pf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) bool low_water_triggered:1; /* A dm event has been sent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) bool suspended:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) bool out_of_data_space:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) struct dm_bio_prison *prison;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) struct dm_kcopyd_client *copier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct work_struct worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) struct workqueue_struct *wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct throttle throttle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) struct delayed_work waker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct delayed_work no_space_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) unsigned long last_commit_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) unsigned ref_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct bio_list deferred_flush_bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) struct bio_list deferred_flush_completions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) struct list_head prepared_mappings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) struct list_head prepared_discards;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) struct list_head prepared_discards_pt2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) struct list_head active_thins;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct dm_deferred_set *shared_read_ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct dm_deferred_set *all_io_ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct dm_thin_new_mapping *next_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) process_bio_fn process_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) process_bio_fn process_discard;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) process_cell_fn process_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) process_cell_fn process_discard_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) process_mapping_fn process_prepared_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) process_mapping_fn process_prepared_discard;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) process_mapping_fn process_prepared_discard_pt2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) struct dm_bio_prison_cell **cell_sort_array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) mempool_t mapping_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) struct bio flush_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static void metadata_operation_failed(struct pool *pool, const char *op, int r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) static enum pool_mode get_pool_mode(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) return pool->pf.mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) static void notify_of_pool_mode_change(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) const char *descs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) "write",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) "out-of-data-space",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) "read-only",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) "read-only",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) "fail"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) const char *extra_desc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) enum pool_mode mode = get_pool_mode(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (mode == PM_OUT_OF_DATA_SPACE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (!pool->pf.error_if_no_space)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) extra_desc = " (queue IO)";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) extra_desc = " (error IO)";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) dm_table_event(pool->ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) DMINFO("%s: switching pool to %s%s mode",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) dm_device_name(pool->pool_md),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) descs[(int)mode], extra_desc ? : "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * Target context for a pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) struct pool_c {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) struct dm_target *ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) struct pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) struct dm_dev *data_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) struct dm_dev *metadata_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) dm_block_t low_water_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) struct pool_features requested_pf; /* Features requested during table load */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) struct pool_features adjusted_pf; /* Features used after adjusting for constituent devices */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * Target context for a thin.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) struct thin_c {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) struct dm_dev *pool_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) struct dm_dev *origin_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) sector_t origin_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) dm_thin_id dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) struct dm_thin_device *td;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) struct mapped_device *thin_md;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) bool requeue_mode:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct list_head deferred_cells;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) struct bio_list deferred_bio_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) struct bio_list retry_on_resume_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) struct rb_root sort_bio_list; /* sorted list of deferred bios */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * Ensures the thin is not destroyed until the worker has finished
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * iterating the active_thins list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) refcount_t refcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) struct completion can_destroy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) static bool block_size_is_power_of_two(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return pool->sectors_per_block_shift >= 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) static sector_t block_to_sectors(struct pool *pool, dm_block_t b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) return block_size_is_power_of_two(pool) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) (b << pool->sectors_per_block_shift) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) (b * pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) struct discard_op {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct bio *parent_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) static void begin_discard(struct discard_op *op, struct thin_c *tc, struct bio *parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) BUG_ON(!parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) op->tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) blk_start_plug(&op->plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) op->parent_bio = parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) op->bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) static int issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t data_e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) struct thin_c *tc = op->tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) sector_t s = block_to_sectors(tc->pool, data_b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) sector_t len = block_to_sectors(tc->pool, data_e - data_b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return __blkdev_issue_discard(tc->pool_dev->bdev, s, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) GFP_NOWAIT, 0, &op->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static void end_discard(struct discard_op *op, int r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (op->bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * Even if one of the calls to issue_discard failed, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * need to wait for the chain to complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) bio_chain(op->bio, op->parent_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) bio_set_op_attrs(op->bio, REQ_OP_DISCARD, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) submit_bio(op->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) blk_finish_plug(&op->plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * Even if r is set, there could be sub discards in flight that we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * need to wait for.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (r && !op->parent_bio->bi_status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) op->parent_bio->bi_status = errno_to_blk_status(r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) bio_endio(op->parent_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * wake_worker() is used when new work is queued and when pool_resume is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * ready to continue deferred IO processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) static void wake_worker(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) queue_work(pool->wq, &pool->worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) struct dm_bio_prison_cell **cell_result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) struct dm_bio_prison_cell *cell_prealloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * Allocate a cell from the prison's mempool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * This might block but it can't fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * We reused an old cell; we can get rid of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * the new one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) dm_bio_prison_free_cell(pool->prison, cell_prealloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) static void cell_release(struct pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) struct dm_bio_prison_cell *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct bio_list *bios)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) dm_cell_release(pool->prison, cell, bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) dm_bio_prison_free_cell(pool->prison, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) static void cell_visit_release(struct pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) void (*fn)(void *, struct dm_bio_prison_cell *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) void *context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) dm_cell_visit_release(pool->prison, fn, context, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) dm_bio_prison_free_cell(pool->prison, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) static void cell_release_no_holder(struct pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) struct dm_bio_prison_cell *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) struct bio_list *bios)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) dm_cell_release_no_holder(pool->prison, cell, bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) dm_bio_prison_free_cell(pool->prison, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) static void cell_error_with_code(struct pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) struct dm_bio_prison_cell *cell, blk_status_t error_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) dm_cell_error(pool->prison, cell, error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) dm_bio_prison_free_cell(pool->prison, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static blk_status_t get_pool_io_error_code(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return pool->out_of_data_space ? BLK_STS_NOSPC : BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) cell_error_with_code(pool, cell, get_pool_io_error_code(pool));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) cell_error_with_code(pool, cell, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) cell_error_with_code(pool, cell, BLK_STS_DM_REQUEUE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * A global list of pools that uses a struct mapped_device as a key.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) static struct dm_thin_pool_table {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) struct mutex mutex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) struct list_head pools;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) } dm_thin_pool_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) static void pool_table_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) mutex_init(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) INIT_LIST_HEAD(&dm_thin_pool_table.pools);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) static void pool_table_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) mutex_destroy(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) static void __pool_table_insert(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) list_add(&pool->list, &dm_thin_pool_table.pools);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) static void __pool_table_remove(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) list_del(&pool->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) static struct pool *__pool_table_lookup(struct mapped_device *md)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) struct pool *pool = NULL, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (tmp->pool_md == md) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) pool = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) return pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) struct pool *pool = NULL, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) if (tmp->md_dev == md_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) pool = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) return pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) struct dm_thin_endio_hook {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) struct dm_deferred_entry *shared_read_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) struct dm_deferred_entry *all_io_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) struct dm_thin_new_mapping *overwrite_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) struct rb_node rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) struct dm_bio_prison_cell *cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) static void __merge_bio_list(struct bio_list *bios, struct bio_list *master)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) bio_list_merge(bios, master);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) bio_list_init(master);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) static void error_bio_list(struct bio_list *bios, blk_status_t error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) while ((bio = bio_list_pop(bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) bio->bi_status = error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) blk_status_t error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) __merge_bio_list(&bios, master);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) error_bio_list(&bios, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) static void requeue_deferred_cells(struct thin_c *tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) struct list_head cells;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct dm_bio_prison_cell *cell, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) INIT_LIST_HEAD(&cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) list_splice_init(&tc->deferred_cells, &cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) list_for_each_entry_safe(cell, tmp, &cells, user_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) cell_requeue(pool, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) static void requeue_io(struct thin_c *tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) __merge_bio_list(&bios, &tc->deferred_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) __merge_bio_list(&bios, &tc->retry_on_resume_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) error_bio_list(&bios, BLK_STS_DM_REQUEUE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) requeue_deferred_cells(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) static void error_retry_list_with_code(struct pool *pool, blk_status_t error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) list_for_each_entry_rcu(tc, &pool->active_thins, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) error_thin_bio_list(tc, &tc->retry_on_resume_list, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) static void error_retry_list(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) error_retry_list_with_code(pool, get_pool_io_error_code(pool));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * This section of code contains the logic for processing a thin device's IO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * Much of the code depends on pool object resources (lists, workqueues, etc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * but most is exclusively called from the thin target rather than the thin-pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * target.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) sector_t block_nr = bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (block_size_is_power_of_two(pool))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) block_nr >>= pool->sectors_per_block_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) (void) sector_div(block_nr, pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) return block_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) * Returns the _complete_ blocks that this bio covers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) static void get_bio_block_range(struct thin_c *tc, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) dm_block_t *begin, dm_block_t *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) sector_t b = bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) b += pool->sectors_per_block - 1ull; /* so we round up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) if (block_size_is_power_of_two(pool)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) b >>= pool->sectors_per_block_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) e >>= pool->sectors_per_block_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) (void) sector_div(b, pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) (void) sector_div(e, pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (e < b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) /* Can happen if the bio is within a single block. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) e = b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) *begin = b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) *end = e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) sector_t bi_sector = bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) bio_set_dev(bio, tc->pool_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (block_size_is_power_of_two(pool))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) bio->bi_iter.bi_sector =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) (block << pool->sectors_per_block_shift) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) (bi_sector & (pool->sectors_per_block - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) sector_div(bi_sector, pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) static void remap_to_origin(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) bio_set_dev(bio, tc->origin_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) return op_is_flush(bio->bi_opf) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) dm_thin_changed_this_transaction(tc->td);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) static void inc_all_io_entry(struct pool *pool, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) struct dm_thin_endio_hook *h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) if (bio_op(bio) == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) static void issue(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) if (!bio_triggers_commit(tc, bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * Complete bio with an error if earlier I/O caused changes to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) * the metadata that can't be committed e.g, due to I/O errors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * on the metadata device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (dm_thin_aborted_changes(tc->td)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * Batch together any bios that trigger commits and then issue a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * single commit for them in process_deferred_bios().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) spin_lock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) bio_list_add(&pool->deferred_flush_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) spin_unlock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) remap_to_origin(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) issue(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) static void remap_and_issue(struct thin_c *tc, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) dm_block_t block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) remap(tc, bio, block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) issue(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * Bio endio functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) struct dm_thin_new_mapping {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) bool pass_discard:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) bool maybe_shared:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) * Track quiescing, copying and zeroing preparation actions. When this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) * counter hits zero the block is prepared and can be inserted into the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) * btree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) atomic_t prepare_actions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) blk_status_t status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) dm_block_t virt_begin, virt_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) dm_block_t data_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) struct dm_bio_prison_cell *cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) * If the bio covers the whole area of a block then we can avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) * zeroing or copying. Instead this bio is hooked. The bio will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) * still be in the cell, so care has to be taken to avoid issuing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) * the bio twice.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) bio_end_io_t *saved_bi_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) struct pool *pool = m->tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) if (atomic_dec_and_test(&m->prepare_actions)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) list_add_tail(&m->list, &pool->prepared_mappings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) wake_worker(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) static void complete_mapping_preparation(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) struct pool *pool = m->tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) __complete_mapping_preparation(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) static void copy_complete(int read_err, unsigned long write_err, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) struct dm_thin_new_mapping *m = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) m->status = read_err || write_err ? BLK_STS_IOERR : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) complete_mapping_preparation(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) static void overwrite_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) struct dm_thin_new_mapping *m = h->overwrite_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) bio->bi_end_io = m->saved_bi_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) m->status = bio->bi_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) complete_mapping_preparation(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * Workqueue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * Prepared mapping jobs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * This sends the bios in the cell, except the original holder, back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * to the deferred_bios list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) int has_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) spin_lock_irqsave(&tc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) has_work = !bio_list_empty(&tc->deferred_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) spin_unlock_irqrestore(&tc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) if (has_work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) wake_worker(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) static void thin_defer_bio(struct thin_c *tc, struct bio *bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) struct remap_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) struct bio_list defer_bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) struct bio_list issue_bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) static void __inc_remap_and_issue_cell(void *context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) struct remap_info *info = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) while ((bio = bio_list_pop(&cell->bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) bio_list_add(&info->defer_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) inc_all_io_entry(info->tc->pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) * We can't issue the bios with the bio prison lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) * held, so we add them to a list to issue on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) * return from this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) bio_list_add(&info->issue_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) static void inc_remap_and_issue_cell(struct thin_c *tc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) struct dm_bio_prison_cell *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) dm_block_t block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) struct remap_info info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) info.tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) bio_list_init(&info.defer_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) bio_list_init(&info.issue_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * We have to be careful to inc any bios we're about to issue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * before the cell is released, and avoid a race with new bios
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) * being added to the cell.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) cell_visit_release(tc->pool, __inc_remap_and_issue_cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) &info, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) while ((bio = bio_list_pop(&info.defer_bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) thin_defer_bio(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) while ((bio = bio_list_pop(&info.issue_bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) remap_and_issue(info.tc, bio, block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) cell_error(m->tc->pool, m->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) list_del(&m->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) mempool_free(m, &m->tc->pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) static void complete_overwrite_bio(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) * If the bio has the REQ_FUA flag set we must commit the metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) * before signaling its completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) if (!bio_triggers_commit(tc, bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) * Complete bio with an error if earlier I/O caused changes to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) * metadata that can't be committed, e.g, due to I/O errors on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * metadata device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) if (dm_thin_aborted_changes(tc->td)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) * Batch together any bios that trigger commits and then issue a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) * single commit for them in process_deferred_bios().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) spin_lock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) bio_list_add(&pool->deferred_flush_completions, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) spin_unlock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) static void process_prepared_mapping(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) struct thin_c *tc = m->tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) struct bio *bio = m->bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) if (m->status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) cell_error(pool, m->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) * Commit the prepared block into the mapping btree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) * Any I/O for this block arriving after this point will get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) * remapped to it directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) metadata_operation_failed(pool, "dm_thin_insert_block", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) cell_error(pool, m->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) * Release any bios held while the block was being provisioned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) * If we are processing a write bio that completely covers the block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) * we already processed it so can ignore it now when processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) * the bios in the cell.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) if (bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) inc_remap_and_issue_cell(tc, m->cell, m->data_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) complete_overwrite_bio(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) inc_all_io_entry(tc->pool, m->cell->holder);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) remap_and_issue(tc, m->cell->holder, m->data_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) inc_remap_and_issue_cell(tc, m->cell, m->data_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) list_del(&m->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) mempool_free(m, &pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) static void free_discard_mapping(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) struct thin_c *tc = m->tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) if (m->cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) cell_defer_no_holder(tc, m->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) mempool_free(m, &tc->pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) bio_io_error(m->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) free_discard_mapping(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) static void process_prepared_discard_success(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) bio_endio(m->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) free_discard_mapping(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) struct thin_c *tc = m->tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) metadata_operation_failed(tc->pool, "dm_thin_remove_range", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) bio_io_error(m->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) bio_endio(m->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) cell_defer_no_holder(tc, m->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) mempool_free(m, &tc->pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) struct bio *discard_parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) * We've already unmapped this range of blocks, but before we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) * passdown we have to check that these blocks are now unused.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) bool shared = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) struct thin_c *tc = m->tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) struct discard_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) begin_discard(&op, tc, discard_parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) while (b != end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) /* find start of unmapped run */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) for (; b < end; b++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) r = dm_pool_block_is_shared(pool->pmd, b, &shared);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) if (!shared)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) if (b == end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) /* find end of run */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) for (e = b + 1; e != end; e++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) r = dm_pool_block_is_shared(pool->pmd, e, &shared);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) if (shared)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) r = issue_discard(&op, b, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) b = e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) end_discard(&op, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) static void queue_passdown_pt2(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) struct pool *pool = m->tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) list_add_tail(&m->list, &pool->prepared_discards_pt2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) wake_worker(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) static void passdown_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) * It doesn't matter if the passdown discard failed, we still want
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) * to unmap (we ignore err).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) queue_passdown_pt2(bio->bi_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) struct thin_c *tc = m->tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) struct bio *discard_parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) * Only this thread allocates blocks, so we can be sure that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) * newly unmapped blocks will not be allocated before the end of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) * the function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) metadata_operation_failed(pool, "dm_thin_remove_range", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) bio_io_error(m->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) cell_defer_no_holder(tc, m->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) mempool_free(m, &pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) * Increment the unmapped blocks. This prevents a race between the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) * passdown io and reallocation of freed blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) metadata_operation_failed(pool, "dm_pool_inc_data_range", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) bio_io_error(m->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) cell_defer_no_holder(tc, m->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) mempool_free(m, &pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) discard_parent = bio_alloc(GFP_NOIO, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) if (!discard_parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) DMWARN("%s: unable to allocate top level discard bio for passdown. Skipping passdown.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) dm_device_name(tc->pool->pool_md));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) queue_passdown_pt2(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) discard_parent->bi_end_io = passdown_endio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) discard_parent->bi_private = m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) if (m->maybe_shared)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) passdown_double_checking_shared_status(m, discard_parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) struct discard_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) begin_discard(&op, tc, discard_parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) r = issue_discard(&op, m->data_block, data_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) end_discard(&op, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) struct thin_c *tc = m->tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) * The passdown has completed, so now we can decrement all those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) * unmapped blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) r = dm_pool_dec_data_range(pool->pmd, m->data_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) m->data_block + (m->virt_end - m->virt_begin));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) metadata_operation_failed(pool, "dm_pool_dec_data_range", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) bio_io_error(m->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) bio_endio(m->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) cell_defer_no_holder(tc, m->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) mempool_free(m, &pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) static void process_prepared(struct pool *pool, struct list_head *head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) process_mapping_fn *fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) struct list_head maps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) struct dm_thin_new_mapping *m, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) INIT_LIST_HEAD(&maps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) spin_lock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) list_splice_init(head, &maps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) spin_unlock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) list_for_each_entry_safe(m, tmp, &maps, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) (*fn)(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) * Deferred bio jobs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) static int io_overlaps_block(struct pool *pool, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) return bio->bi_iter.bi_size ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) (pool->sectors_per_block << SECTOR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) static int io_overwrites_block(struct pool *pool, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) return (bio_data_dir(bio) == WRITE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) io_overlaps_block(pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) bio_end_io_t *fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) *save = bio->bi_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) bio->bi_end_io = fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) static int ensure_next_mapping(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) if (pool->next_mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) pool->next_mapping = mempool_alloc(&pool->mapping_pool, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) return pool->next_mapping ? 0 : -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) struct dm_thin_new_mapping *m = pool->next_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) BUG_ON(!pool->next_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) memset(m, 0, sizeof(struct dm_thin_new_mapping));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) INIT_LIST_HEAD(&m->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) m->bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) pool->next_mapping = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) return m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) sector_t begin, sector_t end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) struct dm_io_region to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) to.bdev = tc->pool_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) to.sector = begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) to.count = end - begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) dm_block_t data_begin,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) struct dm_thin_new_mapping *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) h->overwrite_mapping = m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) m->bio = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) inc_all_io_entry(pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) remap_and_issue(tc, bio, data_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) * A partial copy also needs to zero the uncopied region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) struct dm_dev *origin, dm_block_t data_origin,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) dm_block_t data_dest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) struct dm_bio_prison_cell *cell, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) sector_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) struct dm_thin_new_mapping *m = get_next_mapping(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) m->tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) m->virt_begin = virt_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) m->virt_end = virt_block + 1u;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) m->data_block = data_dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) m->cell = cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) * quiesce action + copy action + an extra reference held for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) * duration of this function (we may need to inc later for a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) * partial zero).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) atomic_set(&m->prepare_actions, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) complete_mapping_preparation(m); /* already quiesced */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) * IO to pool_dev remaps to the pool target's data_dev.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) * If the whole block of data is being overwritten, we can issue the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) * bio immediately. Otherwise we use kcopyd to clone the data first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) if (io_overwrites_block(pool, bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) remap_and_issue_overwrite(tc, bio, data_dest, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) struct dm_io_region from, to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) from.bdev = origin->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) from.sector = data_origin * pool->sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) from.count = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) to.bdev = tc->pool_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) to.sector = data_dest * pool->sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) to.count = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) dm_kcopyd_copy(pool->copier, &from, 1, &to,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) 0, copy_complete, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) * Do we need to zero a tail region?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) atomic_inc(&m->prepare_actions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) ll_zero(tc, m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) data_dest * pool->sectors_per_block + len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) (data_dest + 1) * pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) complete_mapping_preparation(m); /* drop our ref */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) dm_block_t data_origin, dm_block_t data_dest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) struct dm_bio_prison_cell *cell, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) schedule_copy(tc, virt_block, tc->pool_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) data_origin, data_dest, cell, bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) tc->pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) dm_block_t data_block, struct dm_bio_prison_cell *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) struct dm_thin_new_mapping *m = get_next_mapping(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) atomic_set(&m->prepare_actions, 1); /* no need to quiesce */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) m->tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) m->virt_begin = virt_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) m->virt_end = virt_block + 1u;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) m->data_block = data_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) m->cell = cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) * If the whole block of data is being overwritten or we are not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) * zeroing pre-existing data, we can issue the bio immediately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) * Otherwise we use kcopyd to zero the data first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) if (pool->pf.zero_new_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (io_overwrites_block(pool, bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) remap_and_issue_overwrite(tc, bio, data_block, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) ll_zero(tc, m, data_block * pool->sectors_per_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) (data_block + 1) * pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) process_prepared_mapping(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) dm_block_t data_dest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) struct dm_bio_prison_cell *cell, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) sector_t virt_block_begin = virt_block * pool->sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) if (virt_block_end <= tc->origin_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) schedule_copy(tc, virt_block, tc->origin_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) virt_block, data_dest, cell, bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) else if (virt_block_begin < tc->origin_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) schedule_copy(tc, virt_block, tc->origin_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) virt_block, data_dest, cell, bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) tc->origin_size - virt_block_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) schedule_zero(tc, virt_block, data_dest, cell, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) static void requeue_bios(struct pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) static bool is_read_only_pool_mode(enum pool_mode mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) return (mode == PM_OUT_OF_METADATA_SPACE || mode == PM_READ_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) static bool is_read_only(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) return is_read_only_pool_mode(get_pool_mode(pool));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) static void check_for_metadata_space(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) const char *ooms_reason = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) dm_block_t nr_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) ooms_reason = "Could not get free metadata blocks";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) else if (!nr_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) ooms_reason = "No free metadata blocks";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) if (ooms_reason && !is_read_only(pool)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) DMERR("%s", ooms_reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) set_pool_mode(pool, PM_OUT_OF_METADATA_SPACE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) static void check_for_data_space(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) dm_block_t nr_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) r = dm_pool_get_free_block_count(pool->pmd, &nr_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) if (nr_free) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) set_pool_mode(pool, PM_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) requeue_bios(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) * A non-zero return indicates read_only or fail_io mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) * Many callers don't care about the return value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) static int commit(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) r = dm_pool_commit_metadata(pool->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) metadata_operation_failed(pool, "dm_pool_commit_metadata", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) check_for_metadata_space(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) check_for_data_space(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) DMWARN("%s: reached low water mark for data device: sending event.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) dm_device_name(pool->pool_md));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) spin_lock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) pool->low_water_triggered = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) spin_unlock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) dm_table_event(pool->ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) dm_block_t free_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) if (WARN_ON(get_pool_mode(pool) != PM_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) check_low_water_mark(pool, free_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) if (!free_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) * Try to commit to see if that will free up some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) * more space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) r = commit(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) if (!free_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) r = dm_pool_alloc_data_block(pool->pmd, result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) if (r == -ENOSPC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) metadata_operation_failed(pool, "dm_pool_alloc_data_block", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) r = dm_pool_get_free_metadata_block_count(pool->pmd, &free_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) metadata_operation_failed(pool, "dm_pool_get_free_metadata_block_count", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) if (!free_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) /* Let's commit before we use up the metadata reserve. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) r = commit(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) * If we have run out of space, queue bios until the device is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) * resumed, presumably after having been reloaded with more space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) static void retry_on_resume(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) struct thin_c *tc = h->tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) bio_list_add(&tc->retry_on_resume_list, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) static blk_status_t should_error_unserviceable_bio(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) enum pool_mode m = get_pool_mode(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) switch (m) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) case PM_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) /* Shouldn't get here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) case PM_OUT_OF_DATA_SPACE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) return pool->pf.error_if_no_space ? BLK_STS_NOSPC : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) case PM_OUT_OF_METADATA_SPACE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) case PM_READ_ONLY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) case PM_FAIL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) /* Shouldn't get here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) blk_status_t error = should_error_unserviceable_bio(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) bio->bi_status = error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) retry_on_resume(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) blk_status_t error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) error = should_error_unserviceable_bio(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) cell_error_with_code(pool, cell, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) cell_release(pool, cell, &bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) while ((bio = bio_list_pop(&bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) retry_on_resume(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) static void process_discard_cell_no_passdown(struct thin_c *tc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) struct dm_bio_prison_cell *virt_cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) struct dm_thin_new_mapping *m = get_next_mapping(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) * We don't need to lock the data blocks, since there's no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) * passdown. We only lock data blocks for allocation and breaking sharing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) m->tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) m->virt_begin = virt_cell->key.block_begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) m->virt_end = virt_cell->key.block_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) m->cell = virt_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) m->bio = virt_cell->holder;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) pool->process_prepared_discard(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) bool maybe_shared;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) struct dm_cell_key data_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) struct dm_bio_prison_cell *data_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) struct dm_thin_new_mapping *m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) dm_block_t virt_begin, virt_end, data_begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) while (begin != end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) r = ensure_next_mapping(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) /* we did our best */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) &data_begin, &maybe_shared);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) * Silently fail, letting any mappings we've
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) * created complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) build_key(tc->td, PHYSICAL, data_begin, data_begin + (virt_end - virt_begin), &data_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) /* contention, we'll give up with this range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) begin = virt_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) * IO may still be going to the destination block. We must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) * quiesce before we can do the removal.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) m = get_next_mapping(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) m->tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) m->maybe_shared = maybe_shared;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) m->virt_begin = virt_begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) m->virt_end = virt_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) m->data_block = data_begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) m->cell = data_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) m->bio = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) * The parent bio must not complete before sub discard bios are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) * chained to it (see end_discard's bio_chain)!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) * This per-mapping bi_remaining increment is paired with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) * the implicit decrement that occurs via bio_endio() in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) * end_discard().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) bio_inc_remaining(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) pool->process_prepared_discard(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) begin = virt_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) struct bio *bio = virt_cell->holder;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) * The virt_cell will only get freed once the origin bio completes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) * This means it will remain locked while all the individual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) * passdown bios are in flight.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) h->cell = virt_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) * We complete the bio now, knowing that the bi_remaining field
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) * will prevent completion until the sub range discards have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) * completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) static void process_discard_bio(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) dm_block_t begin, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) struct dm_cell_key virt_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) struct dm_bio_prison_cell *virt_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) get_bio_block_range(tc, bio, &begin, &end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) if (begin == end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) * The discard covers less than a block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) build_key(tc->td, VIRTUAL, begin, end, &virt_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) if (bio_detain(tc->pool, &virt_key, bio, &virt_cell))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) * Potential starvation issue: We're relying on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) * fs/application being well behaved, and not trying to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) * send IO to a region at the same time as discarding it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) * If they do this persistently then it's possible this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) * cell will never be granted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) tc->pool->process_discard_cell(tc, virt_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) struct dm_cell_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) struct dm_thin_lookup_result *lookup_result,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) dm_block_t data_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) r = alloc_data_block(tc, &data_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) switch (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) schedule_internal_copy(tc, block, lookup_result->block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) data_block, cell, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) case -ENOSPC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) retry_bios_on_resume(pool, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) __func__, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) cell_error(pool, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) static void __remap_and_issue_shared_cell(void *context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) struct remap_info *info = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) while ((bio = bio_list_pop(&cell->bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) if (bio_data_dir(bio) == WRITE || op_is_flush(bio->bi_opf) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) bio_op(bio) == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) bio_list_add(&info->defer_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) inc_all_io_entry(info->tc->pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) bio_list_add(&info->issue_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) static void remap_and_issue_shared_cell(struct thin_c *tc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) struct dm_bio_prison_cell *cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) dm_block_t block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) struct remap_info info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) info.tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) bio_list_init(&info.defer_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) bio_list_init(&info.issue_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) cell_visit_release(tc->pool, __remap_and_issue_shared_cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) &info, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) while ((bio = bio_list_pop(&info.defer_bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) thin_defer_bio(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) while ((bio = bio_list_pop(&info.issue_bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) remap_and_issue(tc, bio, block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) static void process_shared_bio(struct thin_c *tc, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) dm_block_t block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) struct dm_thin_lookup_result *lookup_result,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) struct dm_bio_prison_cell *virt_cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) struct dm_bio_prison_cell *data_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) struct dm_cell_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) * If cell is already occupied, then sharing is already in the process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) * of being broken so we have nothing further to do here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) build_data_key(tc->td, lookup_result->block, &key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) if (bio_detain(pool, &key, bio, &data_cell)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) cell_defer_no_holder(tc, virt_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) break_sharing(tc, bio, block, &key, lookup_result, data_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) cell_defer_no_holder(tc, virt_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) inc_all_io_entry(pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) remap_and_issue(tc, bio, lookup_result->block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) remap_and_issue_shared_cell(tc, data_cell, lookup_result->block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) dm_block_t data_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) * Remap empty bios (flushes) immediately, without provisioning.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) if (!bio->bi_iter.bi_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) inc_all_io_entry(pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) cell_defer_no_holder(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) remap_and_issue(tc, bio, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) * Fill read bios with zeroes and complete them immediately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) if (bio_data_dir(bio) == READ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) zero_fill_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) cell_defer_no_holder(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) r = alloc_data_block(tc, &data_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) switch (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) if (tc->origin_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) schedule_external_copy(tc, block, data_block, cell, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) schedule_zero(tc, block, data_block, cell, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) case -ENOSPC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) retry_bios_on_resume(pool, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) __func__, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) cell_error(pool, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) struct bio *bio = cell->holder;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) dm_block_t block = get_bio_block(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) struct dm_thin_lookup_result lookup_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) if (tc->requeue_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) cell_requeue(pool, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) switch (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) if (lookup_result.shared)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) process_shared_bio(tc, bio, block, &lookup_result, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) inc_all_io_entry(pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) remap_and_issue(tc, bio, lookup_result.block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) inc_remap_and_issue_cell(tc, cell, lookup_result.block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) case -ENODATA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) if (bio_data_dir(bio) == READ && tc->origin_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) inc_all_io_entry(pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) cell_defer_no_holder(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) if (bio_end_sector(bio) <= tc->origin_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) remap_to_origin_and_issue(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) else if (bio->bi_iter.bi_sector < tc->origin_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) zero_fill_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) remap_to_origin_and_issue(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) zero_fill_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) provision_block(tc, bio, block, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) __func__, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) cell_defer_no_holder(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) static void process_bio(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) dm_block_t block = get_bio_block(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) struct dm_bio_prison_cell *cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) struct dm_cell_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) * If cell is already occupied, then the block is already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) * being provisioned so we have nothing further to do here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) build_virtual_key(tc->td, block, &key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) if (bio_detain(pool, &key, bio, &cell))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) process_cell(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) static void __process_bio_read_only(struct thin_c *tc, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) int rw = bio_data_dir(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) dm_block_t block = get_bio_block(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) struct dm_thin_lookup_result lookup_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) switch (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) handle_unserviceable_bio(tc->pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) if (cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) cell_defer_no_holder(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) inc_all_io_entry(tc->pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) remap_and_issue(tc, bio, lookup_result.block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) if (cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) inc_remap_and_issue_cell(tc, cell, lookup_result.block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) case -ENODATA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) if (cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) cell_defer_no_holder(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) if (rw != READ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) handle_unserviceable_bio(tc->pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) if (tc->origin_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) inc_all_io_entry(tc->pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) remap_to_origin_and_issue(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) zero_fill_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) __func__, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) if (cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) cell_defer_no_holder(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) static void process_bio_read_only(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) __process_bio_read_only(tc, bio, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) __process_bio_read_only(tc, cell->holder, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) static void process_bio_success(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) static void process_bio_fail(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) cell_success(tc->pool, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) cell_error(tc->pool, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) * FIXME: should we also commit due to size of transaction, measured in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) * metadata blocks?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) static int need_commit_due_to_time(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) return !time_in_range(jiffies, pool->last_commit_jiffies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) pool->last_commit_jiffies + COMMIT_PERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) #define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) #define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) struct rb_node **rbp, *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) struct dm_thin_endio_hook *pbd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) sector_t bi_sector = bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) rbp = &tc->sort_bio_list.rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) while (*rbp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) parent = *rbp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) pbd = thin_pbd(parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) rbp = &(*rbp)->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) rbp = &(*rbp)->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) rb_link_node(&pbd->rb_node, parent, rbp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) rb_insert_color(&pbd->rb_node, &tc->sort_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) static void __extract_sorted_bios(struct thin_c *tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) struct dm_thin_endio_hook *pbd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) pbd = thin_pbd(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) bio = thin_bio(pbd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) bio_list_add(&tc->deferred_bio_list, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) rb_erase(&pbd->rb_node, &tc->sort_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) static void __sort_thin_deferred_bios(struct thin_c *tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) bio_list_merge(&bios, &tc->deferred_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) bio_list_init(&tc->deferred_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) /* Sort deferred_bio_list using rb-tree */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) while ((bio = bio_list_pop(&bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) __thin_bio_rb_add(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) * Transfer the sorted bios in sort_bio_list back to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) * deferred_bio_list to allow lockless submission of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) * all bios.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) __extract_sorted_bios(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) static void process_thin_deferred_bios(struct thin_c *tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) unsigned count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) if (tc->requeue_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) error_thin_bio_list(tc, &tc->deferred_bio_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) BLK_STS_DM_REQUEUE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) if (bio_list_empty(&tc->deferred_bio_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) __sort_thin_deferred_bios(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) bio_list_merge(&bios, &tc->deferred_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) bio_list_init(&tc->deferred_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) while ((bio = bio_list_pop(&bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) * If we've got no free new_mapping structs, and processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) * this bio might require one, we pause until there are some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) * prepared mappings to process.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) if (ensure_next_mapping(pool)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) bio_list_add(&tc->deferred_bio_list, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) bio_list_merge(&tc->deferred_bio_list, &bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) if (bio_op(bio) == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) pool->process_discard(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) pool->process_bio(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) if ((count++ & 127) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) throttle_work_update(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) dm_pool_issue_prefetches(pool->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) static int cmp_cells(const void *lhs, const void *rhs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) BUG_ON(!lhs_cell->holder);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) BUG_ON(!rhs_cell->holder);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) static unsigned sort_cells(struct pool *pool, struct list_head *cells)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) unsigned count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) struct dm_bio_prison_cell *cell, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) list_for_each_entry_safe(cell, tmp, cells, user_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) if (count >= CELL_SORT_ARRAY_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) pool->cell_sort_array[count++] = cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) list_del(&cell->user_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) static void process_thin_deferred_cells(struct thin_c *tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) struct list_head cells;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) struct dm_bio_prison_cell *cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) unsigned i, j, count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) INIT_LIST_HEAD(&cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) list_splice_init(&tc->deferred_cells, &cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) if (list_empty(&cells))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) count = sort_cells(tc->pool, &cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) for (i = 0; i < count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) cell = pool->cell_sort_array[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) BUG_ON(!cell->holder);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) * If we've got no free new_mapping structs, and processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) * this bio might require one, we pause until there are some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) * prepared mappings to process.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) if (ensure_next_mapping(pool)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) for (j = i; j < count; j++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) list_add(&pool->cell_sort_array[j]->user_list, &cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) list_splice(&cells, &tc->deferred_cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) if (bio_op(cell->holder) == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) pool->process_discard_cell(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) pool->process_cell(tc, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) } while (!list_empty(&cells));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) static void thin_get(struct thin_c *tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) static void thin_put(struct thin_c *tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) * We can't hold rcu_read_lock() around code that can block. So we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) * find a thin with the rcu lock held; bump a refcount; then drop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) * the lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) static struct thin_c *get_first_thin(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) struct thin_c *tc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) if (!list_empty(&pool->active_thins)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) thin_get(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) return tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) struct thin_c *old_tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) thin_get(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) thin_put(old_tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) return tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) thin_put(old_tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) static void process_deferred_bios(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) struct bio_list bios, bio_completions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) tc = get_first_thin(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) while (tc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) process_thin_deferred_cells(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) process_thin_deferred_bios(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) tc = get_next_thin(pool, tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) * If there are any deferred flush bios, we must commit the metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) * before issuing them or signaling their completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) bio_list_init(&bio_completions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) spin_lock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) bio_list_merge(&bios, &pool->deferred_flush_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) bio_list_init(&pool->deferred_flush_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) bio_list_merge(&bio_completions, &pool->deferred_flush_completions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) bio_list_init(&pool->deferred_flush_completions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) spin_unlock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) if (commit(pool)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) bio_list_merge(&bios, &bio_completions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) while ((bio = bio_list_pop(&bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) pool->last_commit_jiffies = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) while ((bio = bio_list_pop(&bio_completions)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) while ((bio = bio_list_pop(&bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) * The data device was flushed as part of metadata commit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) * so complete redundant flushes immediately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) if (bio->bi_opf & REQ_PREFLUSH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) static void do_worker(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) struct pool *pool = container_of(ws, struct pool, worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) throttle_work_start(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) dm_pool_issue_prefetches(pool->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) throttle_work_update(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) throttle_work_update(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) throttle_work_update(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) process_prepared(pool, &pool->prepared_discards_pt2, &pool->process_prepared_discard_pt2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) throttle_work_update(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) process_deferred_bios(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) throttle_work_complete(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) * We want to commit periodically so that not too much
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) * unwritten data builds up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) static void do_waker(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) wake_worker(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) * We're holding onto IO to allow userland time to react. After the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) * timeout either the pool will have been resized (and thus back in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) static void do_no_space_timeout(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) struct pool *pool = container_of(to_delayed_work(ws), struct pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) no_space_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) pool->pf.error_if_no_space = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) notify_of_pool_mode_change(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) error_retry_list_with_code(pool, BLK_STS_NOSPC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) struct pool_work {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) struct work_struct worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) struct completion complete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) static struct pool_work *to_pool_work(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) return container_of(ws, struct pool_work, worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) static void pool_work_complete(struct pool_work *pw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) complete(&pw->complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) static void pool_work_wait(struct pool_work *pw, struct pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) void (*fn)(struct work_struct *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) INIT_WORK_ONSTACK(&pw->worker, fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) init_completion(&pw->complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) queue_work(pool->wq, &pw->worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) wait_for_completion(&pw->complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) struct noflush_work {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) struct pool_work pw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) static struct noflush_work *to_noflush(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) return container_of(to_pool_work(ws), struct noflush_work, pw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) static void do_noflush_start(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) struct noflush_work *w = to_noflush(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) w->tc->requeue_mode = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) requeue_io(w->tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) pool_work_complete(&w->pw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) static void do_noflush_stop(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) struct noflush_work *w = to_noflush(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) w->tc->requeue_mode = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) pool_work_complete(&w->pw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) struct noflush_work w;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) w.tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) pool_work_wait(&w.pw, tc->pool, fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) static bool passdown_enabled(struct pool_c *pt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) return pt->adjusted_pf.discard_passdown;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) static void set_discard_callbacks(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) struct pool_c *pt = pool->ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) if (passdown_enabled(pt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) pool->process_discard_cell = process_discard_cell_passdown;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) pool->process_prepared_discard = process_prepared_discard_passdown_pt1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) pool->process_discard_cell = process_discard_cell_no_passdown;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) pool->process_prepared_discard = process_prepared_discard_no_passdown;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) struct pool_c *pt = pool->ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) bool needs_check = dm_pool_metadata_needs_check(pool->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) enum pool_mode old_mode = get_pool_mode(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) unsigned long no_space_timeout = READ_ONCE(no_space_timeout_secs) * HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) * Never allow the pool to transition to PM_WRITE mode if user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) * intervention is required to verify metadata and data consistency.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) if (new_mode == PM_WRITE && needs_check) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) DMERR("%s: unable to switch pool to write mode until repaired.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) dm_device_name(pool->pool_md));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) if (old_mode != new_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) new_mode = old_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) new_mode = PM_READ_ONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) * If we were in PM_FAIL mode, rollback of metadata failed. We're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) * not going to recover without a thin_repair. So we never let the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) * pool move out of the old mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) if (old_mode == PM_FAIL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) new_mode = old_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) switch (new_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) case PM_FAIL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) dm_pool_metadata_read_only(pool->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) pool->process_bio = process_bio_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) pool->process_discard = process_bio_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) pool->process_cell = process_cell_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) pool->process_discard_cell = process_cell_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) pool->process_prepared_mapping = process_prepared_mapping_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) pool->process_prepared_discard = process_prepared_discard_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) error_retry_list(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) case PM_OUT_OF_METADATA_SPACE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) case PM_READ_ONLY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) dm_pool_metadata_read_only(pool->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) pool->process_bio = process_bio_read_only;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) pool->process_discard = process_bio_success;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) pool->process_cell = process_cell_read_only;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) pool->process_discard_cell = process_cell_success;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) pool->process_prepared_mapping = process_prepared_mapping_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) pool->process_prepared_discard = process_prepared_discard_success;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) error_retry_list(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) case PM_OUT_OF_DATA_SPACE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) * Ideally we'd never hit this state; the low water mark
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) * would trigger userland to extend the pool before we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) * completely run out of data space. However, many small
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) * IOs to unprovisioned space can consume data space at an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) * alarming rate. Adjust your low water mark if you're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) * frequently seeing this mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) pool->out_of_data_space = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) pool->process_bio = process_bio_read_only;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) pool->process_discard = process_discard_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) pool->process_cell = process_cell_read_only;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) pool->process_prepared_mapping = process_prepared_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) set_discard_callbacks(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) if (!pool->pf.error_if_no_space && no_space_timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) case PM_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) if (old_mode == PM_OUT_OF_DATA_SPACE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) cancel_delayed_work_sync(&pool->no_space_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) pool->out_of_data_space = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) dm_pool_metadata_read_write(pool->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) pool->process_bio = process_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) pool->process_discard = process_discard_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) pool->process_cell = process_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) pool->process_prepared_mapping = process_prepared_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) set_discard_callbacks(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) pool->pf.mode = new_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) * The pool mode may have changed, sync it so bind_control_target()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) * doesn't cause an unexpected mode transition on resume.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) pt->adjusted_pf.mode = new_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) if (old_mode != new_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) notify_of_pool_mode_change(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) static void abort_transaction(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) const char *dev_name = dm_device_name(pool->pool_md);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) if (dm_pool_abort_metadata(pool->pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) DMERR("%s: failed to abort metadata transaction", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) set_pool_mode(pool, PM_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) if (dm_pool_metadata_set_needs_check(pool->pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) set_pool_mode(pool, PM_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) static void metadata_operation_failed(struct pool *pool, const char *op, int r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) dm_device_name(pool->pool_md), op, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) abort_transaction(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) set_pool_mode(pool, PM_READ_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) * Mapping functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) * Called only while mapping a thin bio to hand it over to the workqueue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) bio_list_add(&tc->deferred_bio_list, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) wake_worker(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) throttle_lock(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) thin_defer_bio(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) throttle_unlock(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) throttle_lock(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) list_add_tail(&cell->user_list, &tc->deferred_cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) throttle_unlock(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) wake_worker(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) static void thin_hook_bio(struct thin_c *tc, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) h->tc = tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) h->shared_read_entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) h->all_io_entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) h->overwrite_mapping = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) h->cell = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) * Non-blocking function called from the thin target's map function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) static int thin_bio_map(struct dm_target *ti, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) struct thin_c *tc = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) dm_block_t block = get_bio_block(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) struct dm_thin_device *td = tc->td;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) struct dm_thin_lookup_result result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) struct dm_bio_prison_cell *virt_cell, *data_cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) struct dm_cell_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) thin_hook_bio(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) if (tc->requeue_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) bio->bi_status = BLK_STS_DM_REQUEUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) if (get_pool_mode(tc->pool) == PM_FAIL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) thin_defer_bio_with_throttle(tc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) * We must hold the virtual cell before doing the lookup, otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) * there's a race with discard.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) build_virtual_key(tc->td, block, &key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) if (bio_detain(tc->pool, &key, bio, &virt_cell))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) r = dm_thin_find_block(td, block, 0, &result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) * Note that we defer readahead too.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) switch (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) if (unlikely(result.shared)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) * We have a race condition here between the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) * result.shared value returned by the lookup and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) * snapshot creation, which may cause new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) * sharing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) * To avoid this always quiesce the origin before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) * taking the snap. You want to do this anyway to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) * ensure a consistent application view
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) * (i.e. lockfs).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) * More distant ancestors are irrelevant. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) * shared flag will be set in their case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) thin_defer_cell(tc, virt_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) build_data_key(tc->td, result.block, &key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) if (bio_detain(tc->pool, &key, bio, &data_cell)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) cell_defer_no_holder(tc, virt_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) inc_all_io_entry(tc->pool, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) cell_defer_no_holder(tc, data_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) cell_defer_no_holder(tc, virt_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) remap(tc, bio, result.block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) return DM_MAPIO_REMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) case -ENODATA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) case -EWOULDBLOCK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) thin_defer_cell(tc, virt_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) * Must always call bio_io_error on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) * dm_thin_find_block can fail with -EINVAL if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) * pool is switched to fail-io mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) cell_defer_no_holder(tc, virt_cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) static void requeue_bios(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) list_for_each_entry_rcu(tc, &pool->active_thins, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) spin_lock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) bio_list_init(&tc->retry_on_resume_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) spin_unlock_irq(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) * Binding of control targets to a pool object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) static bool data_dev_supports_discard(struct pool_c *pt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) return q && blk_queue_discard(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) static bool is_factor(sector_t block_size, uint32_t n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) return !sector_div(block_size, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) * If discard_passdown was enabled verify that the data device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) * supports discards. Disable discard_passdown if not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) static void disable_passdown_if_not_supported(struct pool_c *pt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) struct block_device *data_bdev = pt->data_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) struct queue_limits *data_limits = &bdev_get_queue(data_bdev)->limits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) const char *reason = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) char buf[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) if (!pt->adjusted_pf.discard_passdown)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) if (!data_dev_supports_discard(pt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) reason = "discard unsupported";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) else if (data_limits->max_discard_sectors < pool->sectors_per_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) reason = "max discard sectors smaller than a block";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) if (reason) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) DMWARN("Data device (%s) %s: Disabling discard passdown.", bdevname(data_bdev, buf), reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) pt->adjusted_pf.discard_passdown = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) static int bind_control_target(struct pool *pool, struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) * We want to make sure that a pool in PM_FAIL mode is never upgraded.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) enum pool_mode old_mode = get_pool_mode(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) enum pool_mode new_mode = pt->adjusted_pf.mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) * Don't change the pool's mode until set_pool_mode() below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) * Otherwise the pool's process_* function pointers may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) * not match the desired pool mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) pt->adjusted_pf.mode = old_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) pool->ti = ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) pool->pf = pt->adjusted_pf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) pool->low_water_blocks = pt->low_water_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) set_pool_mode(pool, new_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) static void unbind_control_target(struct pool *pool, struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) if (pool->ti == ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) pool->ti = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) * Pool creation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) /* Initialize pool features. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) static void pool_features_init(struct pool_features *pf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) pf->mode = PM_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) pf->zero_new_blocks = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) pf->discard_enabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) pf->discard_passdown = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) pf->error_if_no_space = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) static void __pool_destroy(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) __pool_table_remove(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) vfree(pool->cell_sort_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) if (dm_pool_metadata_close(pool->pmd) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) dm_bio_prison_destroy(pool->prison);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) dm_kcopyd_client_destroy(pool->copier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) if (pool->wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) destroy_workqueue(pool->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) if (pool->next_mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) mempool_free(pool->next_mapping, &pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) mempool_exit(&pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) bio_uninit(&pool->flush_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) dm_deferred_set_destroy(pool->shared_read_ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) dm_deferred_set_destroy(pool->all_io_ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) kfree(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) static struct kmem_cache *_new_mapping_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) static struct pool *pool_create(struct mapped_device *pool_md,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) struct block_device *metadata_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) struct block_device *data_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) unsigned long block_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) int read_only, char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) void *err_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) struct pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) struct dm_pool_metadata *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) bool format_device = read_only ? false : true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) if (IS_ERR(pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) *error = "Error creating metadata object";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) return (struct pool *)pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) pool = kzalloc(sizeof(*pool), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) if (!pool) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) *error = "Error allocating memory for pool";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) err_p = ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) goto bad_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) pool->pmd = pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) pool->sectors_per_block = block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) if (block_size & (block_size - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) pool->sectors_per_block_shift = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) pool->sectors_per_block_shift = __ffs(block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) pool->low_water_blocks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) pool_features_init(&pool->pf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) pool->prison = dm_bio_prison_create();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) if (!pool->prison) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) *error = "Error creating pool's bio prison";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) err_p = ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) goto bad_prison;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) if (IS_ERR(pool->copier)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) r = PTR_ERR(pool->copier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) *error = "Error creating pool's kcopyd client";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) err_p = ERR_PTR(r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) goto bad_kcopyd_client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) * Create singlethreaded workqueue that will service all devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) * that use this metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) if (!pool->wq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) *error = "Error creating pool's workqueue";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) err_p = ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) goto bad_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) throttle_init(&pool->throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) INIT_WORK(&pool->worker, do_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) INIT_DELAYED_WORK(&pool->waker, do_waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) spin_lock_init(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) bio_list_init(&pool->deferred_flush_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) bio_list_init(&pool->deferred_flush_completions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) INIT_LIST_HEAD(&pool->prepared_mappings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) INIT_LIST_HEAD(&pool->prepared_discards);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) INIT_LIST_HEAD(&pool->prepared_discards_pt2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) INIT_LIST_HEAD(&pool->active_thins);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) pool->low_water_triggered = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) pool->suspended = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) pool->out_of_data_space = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) bio_init(&pool->flush_bio, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) pool->shared_read_ds = dm_deferred_set_create();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) if (!pool->shared_read_ds) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) *error = "Error creating pool's shared read deferred set";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) err_p = ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) goto bad_shared_read_ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) pool->all_io_ds = dm_deferred_set_create();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) if (!pool->all_io_ds) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) *error = "Error creating pool's all io deferred set";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) err_p = ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) goto bad_all_io_ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) pool->next_mapping = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) r = mempool_init_slab_pool(&pool->mapping_pool, MAPPING_POOL_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) _new_mapping_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) *error = "Error creating pool's mapping mempool";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) err_p = ERR_PTR(r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) goto bad_mapping_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) pool->cell_sort_array =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) vmalloc(array_size(CELL_SORT_ARRAY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) sizeof(*pool->cell_sort_array)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) if (!pool->cell_sort_array) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) *error = "Error allocating cell sort array";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) err_p = ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) goto bad_sort_array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) pool->ref_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) pool->last_commit_jiffies = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) pool->pool_md = pool_md;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) pool->md_dev = metadata_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) pool->data_dev = data_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) __pool_table_insert(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) return pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) bad_sort_array:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) mempool_exit(&pool->mapping_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) bad_mapping_pool:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) dm_deferred_set_destroy(pool->all_io_ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) bad_all_io_ds:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) dm_deferred_set_destroy(pool->shared_read_ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) bad_shared_read_ds:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) destroy_workqueue(pool->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) bad_wq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) dm_kcopyd_client_destroy(pool->copier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) bad_kcopyd_client:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) dm_bio_prison_destroy(pool->prison);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) bad_prison:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) kfree(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) bad_pool:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) if (dm_pool_metadata_close(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) return err_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) static void __pool_inc(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) pool->ref_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) static void __pool_dec(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) BUG_ON(!pool->ref_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) if (!--pool->ref_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) __pool_destroy(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) static struct pool *__pool_find(struct mapped_device *pool_md,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) struct block_device *metadata_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) struct block_device *data_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) unsigned long block_size, int read_only,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) char **error, int *created)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) if (pool) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) if (pool->pool_md != pool_md) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) *error = "metadata device already in use by a pool";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) return ERR_PTR(-EBUSY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) if (pool->data_dev != data_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) *error = "data device already in use by a pool";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) return ERR_PTR(-EBUSY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) __pool_inc(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) pool = __pool_table_lookup(pool_md);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) if (pool) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) if (pool->md_dev != metadata_dev || pool->data_dev != data_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) *error = "different pool cannot replace a pool";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) __pool_inc(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) pool = pool_create(pool_md, metadata_dev, data_dev, block_size, read_only, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) *created = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) return pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) * Pool target methods
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) static void pool_dtr(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) mutex_lock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) unbind_control_target(pt->pool, ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) __pool_dec(pt->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) dm_put_device(ti, pt->metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) dm_put_device(ti, pt->data_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) kfree(pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) mutex_unlock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) unsigned argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) const char *arg_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) static const struct dm_arg _args[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) {0, 4, "Invalid number of pool feature arguments"},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) * No feature arguments supplied.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) if (!as->argc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) r = dm_read_arg_group(_args, as, &argc, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) while (argc && !r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) arg_name = dm_shift_arg(as);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) argc--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) if (!strcasecmp(arg_name, "skip_block_zeroing"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) pf->zero_new_blocks = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) else if (!strcasecmp(arg_name, "ignore_discard"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) pf->discard_enabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) else if (!strcasecmp(arg_name, "no_discard_passdown"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) pf->discard_passdown = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) else if (!strcasecmp(arg_name, "read_only"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) pf->mode = PM_READ_ONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) else if (!strcasecmp(arg_name, "error_if_no_space"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) pf->error_if_no_space = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) ti->error = "Unrecognised pool feature requested";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) static void metadata_low_callback(void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) struct pool *pool = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) DMWARN("%s: reached low water mark for metadata device: sending event.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) dm_device_name(pool->pool_md));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) dm_table_event(pool->ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) * We need to flush the data device **before** committing the metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) * This ensures that the data blocks of any newly inserted mappings are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) * properly written to non-volatile storage and won't be lost in case of a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) * crash.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) * Failure to do so can result in data corruption in the case of internal or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) * external snapshots and in the case of newly provisioned blocks, when block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) * zeroing is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) static int metadata_pre_commit_callback(void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) struct pool *pool = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) struct bio *flush_bio = &pool->flush_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) bio_reset(flush_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) bio_set_dev(flush_bio, pool->data_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) flush_bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) return submit_bio_wait(flush_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) static sector_t get_dev_size(struct block_device *bdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) return i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) static void warn_if_metadata_device_too_big(struct block_device *bdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) sector_t metadata_dev_size = get_dev_size(bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) char buffer[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) bdevname(bdev, buffer), THIN_METADATA_MAX_SECTORS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) static sector_t get_metadata_dev_size(struct block_device *bdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) sector_t metadata_dev_size = get_dev_size(bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) if (metadata_dev_size > THIN_METADATA_MAX_SECTORS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) metadata_dev_size = THIN_METADATA_MAX_SECTORS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) return metadata_dev_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) sector_t metadata_dev_size = get_metadata_dev_size(bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) return metadata_dev_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) * When a metadata threshold is crossed a dm event is triggered, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) * userland should respond by growing the metadata device. We could let
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) * userland set the threshold, like we do with the data threshold, but I'm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) * not sure they know enough to do this well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) static dm_block_t calc_metadata_threshold(struct pool_c *pt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) * 4M is ample for all ops with the possible exception of thin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) * device deletion which is harmless if it fails (just retry the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) * delete after you've grown the device).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) return min((dm_block_t)1024ULL /* 4M */, quarter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) * thin-pool <metadata dev> <data dev>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) * <data block size (sectors)>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) * <low water mark (blocks)>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) * [<#feature args> [<arg>]*]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) * Optional feature arguments are:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) * skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) * ignore_discard: disable discard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) * no_discard_passdown: don't pass discards down to the data device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) * read_only: Don't allow any changes to be made to the pool metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) * error_if_no_space: error IOs, instead of queueing, if no space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) int r, pool_created = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) struct pool_c *pt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) struct pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) struct pool_features pf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) struct dm_arg_set as;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) struct dm_dev *data_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) unsigned long block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) dm_block_t low_water_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) struct dm_dev *metadata_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) fmode_t metadata_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) * FIXME Remove validation from scope of lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) mutex_lock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) if (argc < 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) ti->error = "Invalid argument count";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) as.argc = argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) as.argv = argv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) /* make sure metadata and data are different devices */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) if (!strcmp(argv[0], argv[1])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) ti->error = "Error setting metadata or data device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) * Set default pool features.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) pool_features_init(&pf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) dm_consume_args(&as, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) r = parse_pool_features(&as, &pf, ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) metadata_mode = FMODE_READ | ((pf.mode == PM_READ_ONLY) ? 0 : FMODE_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) ti->error = "Error opening metadata block device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) warn_if_metadata_device_too_big(metadata_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) ti->error = "Error getting data device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) goto out_metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) ti->error = "Invalid block size";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) ti->error = "Invalid low water mark";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) pt = kzalloc(sizeof(*pt), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) if (!pt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev, data_dev->bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) if (IS_ERR(pool)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) r = PTR_ERR(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) goto out_free_pt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) * 'pool_created' reflects whether this is the first table load.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) * Top level discard support is not allowed to be changed after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) * initial load. This would require a pool reload to trigger thin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) * device changes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) ti->error = "Discard support cannot be disabled once enabled";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) goto out_flags_changed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) pt->pool = pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) pt->ti = ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) pt->metadata_dev = metadata_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) pt->data_dev = data_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) pt->low_water_blocks = low_water_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) pt->adjusted_pf = pt->requested_pf = pf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) ti->num_flush_bios = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) * Only need to enable discards if the pool should pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) * them down to the data device. The thin device's discard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) * processing will cause mappings to be removed from the btree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) if (pf.discard_enabled && pf.discard_passdown) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) ti->num_discard_bios = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) * Setting 'discards_supported' circumvents the normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) * stacking of discard limits (this keeps the pool and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) * thin devices' discard limits consistent).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) ti->discards_supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) ti->private = pt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) r = dm_pool_register_metadata_threshold(pt->pool->pmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) calc_metadata_threshold(pt),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) metadata_low_callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) goto out_flags_changed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) dm_pool_register_pre_commit_callback(pool->pmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) metadata_pre_commit_callback, pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) mutex_unlock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) out_flags_changed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) __pool_dec(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) out_free_pt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) kfree(pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) dm_put_device(ti, data_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) out_metadata:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) dm_put_device(ti, metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) mutex_unlock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) static int pool_map(struct dm_target *ti, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) * As this is a singleton target, ti->begin is always zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) spin_lock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) bio_set_dev(bio, pt->data_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) r = DM_MAPIO_REMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) spin_unlock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) sector_t data_size = ti->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) dm_block_t sb_data_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) *need_commit = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) (void) sector_div(data_size, pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) DMERR("%s: failed to retrieve data device size",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) dm_device_name(pool->pool_md));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) if (data_size < sb_data_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465) DMERR("%s: pool target (%llu blocks) too small: expected %llu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) dm_device_name(pool->pool_md),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) (unsigned long long)data_size, sb_data_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) } else if (data_size > sb_data_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) if (dm_pool_metadata_needs_check(pool->pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) DMERR("%s: unable to grow the data device until repaired.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) dm_device_name(pool->pool_md));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) if (sb_data_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) DMINFO("%s: growing the data device from %llu to %llu blocks",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) dm_device_name(pool->pool_md),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) sb_data_size, (unsigned long long)data_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) r = dm_pool_resize_data_dev(pool->pmd, data_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) metadata_operation_failed(pool, "dm_pool_resize_data_dev", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) *need_commit = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) dm_block_t metadata_dev_size, sb_metadata_dev_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) *need_commit = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) DMERR("%s: failed to retrieve metadata device size",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) dm_device_name(pool->pool_md));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) if (metadata_dev_size < sb_metadata_dev_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) DMERR("%s: metadata device (%llu blocks) too small: expected %llu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) dm_device_name(pool->pool_md),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) metadata_dev_size, sb_metadata_dev_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) } else if (metadata_dev_size > sb_metadata_dev_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518) if (dm_pool_metadata_needs_check(pool->pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) DMERR("%s: unable to grow the metadata device until repaired.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) dm_device_name(pool->pool_md));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) warn_if_metadata_device_too_big(pool->md_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525) DMINFO("%s: growing the metadata device from %llu to %llu blocks",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) dm_device_name(pool->pool_md),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) sb_metadata_dev_size, metadata_dev_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529) if (get_pool_mode(pool) == PM_OUT_OF_METADATA_SPACE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) set_pool_mode(pool, PM_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532) r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) *need_commit = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) * Retrieves the number of blocks of the data device from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) * the superblock and compares it to the actual device size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) * thus resizing the data device in case it has grown.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) * This both copes with opening preallocated data devices in the ctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) * being followed by a resume
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551) * -and-
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) * calling the resume method individually after userspace has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) * grown the data device in reaction to a table event.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) static int pool_preresume(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) bool need_commit1, need_commit2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) * Take control of the pool object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) r = bind_control_target(pool, ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) r = maybe_resize_data_dev(ti, &need_commit1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) r = maybe_resize_metadata_dev(ti, &need_commit2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) if (need_commit1 || need_commit2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) (void) commit(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) static void pool_suspend_active_thins(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) /* Suspend all active thin devices */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) tc = get_first_thin(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) while (tc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) dm_internal_suspend_noflush(tc->thin_md);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) tc = get_next_thin(pool, tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) static void pool_resume_active_thins(struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) /* Resume all active thin devices */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) tc = get_first_thin(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) while (tc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) dm_internal_resume(tc->thin_md);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) tc = get_next_thin(pool, tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) static void pool_resume(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) * Must requeue active_thins' bios and then resume
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) * active_thins _before_ clearing 'suspend' flag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) requeue_bios(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) pool_resume_active_thins(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) spin_lock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) pool->low_water_triggered = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) pool->suspended = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) spin_unlock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) do_waker(&pool->waker.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) static void pool_presuspend(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) spin_lock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) pool->suspended = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) spin_unlock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) pool_suspend_active_thins(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) static void pool_presuspend_undo(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) pool_resume_active_thins(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) spin_lock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647) pool->suspended = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) spin_unlock_irq(&pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) static void pool_postsuspend(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) cancel_delayed_work_sync(&pool->waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) cancel_delayed_work_sync(&pool->no_space_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) flush_workqueue(pool->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) (void) commit(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) static int check_arg_count(unsigned argc, unsigned args_required)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) if (argc != args_required) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) DMWARN("Message received with %u arguments instead of %u.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) argc, args_required);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) *dev_id <= MAX_DEV_ID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) if (warning)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) DMWARN("Message received with invalid device id: %s", arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687) dm_thin_id dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) r = check_arg_count(argc, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) r = read_dev_id(argv[1], &dev_id, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) r = dm_pool_create_thin(pool->pmd, dev_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) DMWARN("Creation of new thinly-provisioned device with id %s failed.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) argv[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) dm_thin_id dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) dm_thin_id origin_dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714) r = check_arg_count(argc, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) r = read_dev_id(argv[1], &dev_id, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722) r = read_dev_id(argv[2], &origin_dev_id, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) DMWARN("Creation of new snapshot %s of device %s failed.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) argv[1], argv[2]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) dm_thin_id dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) r = check_arg_count(argc, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) r = read_dev_id(argv[1], &dev_id, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749) r = dm_pool_delete_thin_device(pool->pmd, dev_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) DMWARN("Deletion of thin device %s failed.", argv[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756) static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) dm_thin_id old_id, new_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) r = check_arg_count(argc, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766) DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771) DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) DMWARN("Failed to change transaction id from %s to %s.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) argv[1], argv[2]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) static int process_reserve_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) r = check_arg_count(argc, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) (void) commit(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) r = dm_pool_reserve_metadata_snap(pool->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797) DMWARN("reserve_metadata_snap message failed.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806) r = check_arg_count(argc, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) r = dm_pool_release_metadata_snap(pool->pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812) DMWARN("release_metadata_snap message failed.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) * Messages supported:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) * create_thin <dev_id>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820) * create_snap <dev_id> <origin_id>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) * delete <dev_id>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) * set_transaction_id <current_trans_id> <new_trans_id>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) * reserve_metadata_snap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) * release_metadata_snap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) static int pool_message(struct dm_target *ti, unsigned argc, char **argv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) char *result, unsigned maxlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) int r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) dm_device_name(pool->pool_md));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) if (!strcasecmp(argv[0], "create_thin"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) r = process_create_thin_mesg(argc, argv, pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) else if (!strcasecmp(argv[0], "create_snap"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843) r = process_create_snap_mesg(argc, argv, pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) else if (!strcasecmp(argv[0], "delete"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) r = process_delete_mesg(argc, argv, pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) else if (!strcasecmp(argv[0], "set_transaction_id"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) r = process_set_transaction_id_mesg(argc, argv, pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851) else if (!strcasecmp(argv[0], "reserve_metadata_snap"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) r = process_reserve_metadata_snap_mesg(argc, argv, pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854) else if (!strcasecmp(argv[0], "release_metadata_snap"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) r = process_release_metadata_snap_mesg(argc, argv, pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) if (!r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) (void) commit(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) static void emit_flags(struct pool_features *pf, char *result,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) unsigned sz, unsigned maxlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) unsigned count = !pf->zero_new_blocks + !pf->discard_enabled +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) !pf->discard_passdown + (pf->mode == PM_READ_ONLY) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) pf->error_if_no_space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) DMEMIT("%u ", count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) if (!pf->zero_new_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) DMEMIT("skip_block_zeroing ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) if (!pf->discard_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) DMEMIT("ignore_discard ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) if (!pf->discard_passdown)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) DMEMIT("no_discard_passdown ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) if (pf->mode == PM_READ_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884) DMEMIT("read_only ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) if (pf->error_if_no_space)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) DMEMIT("error_if_no_space ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) * Status line is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) * <transaction id> <used metadata sectors>/<total metadata sectors>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) * <used data sectors>/<total data sectors> <held metadata root>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) * <pool mode> <discard config> <no space config> <needs_check>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) static void pool_status(struct dm_target *ti, status_type_t type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) unsigned status_flags, char *result, unsigned maxlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) unsigned sz = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901) uint64_t transaction_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) dm_block_t nr_free_blocks_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) dm_block_t nr_free_blocks_metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904) dm_block_t nr_blocks_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) dm_block_t nr_blocks_metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) dm_block_t held_root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) enum pool_mode mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908) char buf[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) char buf2[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) case STATUSTYPE_INFO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) if (get_pool_mode(pool) == PM_FAIL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) DMEMIT("Fail");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920) /* Commit to ensure statistics aren't out-of-date */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921) if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) (void) commit(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926) DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927) dm_device_name(pool->pool_md), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933) DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) dm_device_name(pool->pool_md), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) DMERR("%s: dm_pool_get_metadata_dev_size returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) dm_device_name(pool->pool_md), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) DMERR("%s: dm_pool_get_free_block_count returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) dm_device_name(pool->pool_md), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) DMERR("%s: dm_pool_get_data_dev_size returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955) dm_device_name(pool->pool_md), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959) r = dm_pool_get_metadata_snap(pool->pmd, &held_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) DMERR("%s: dm_pool_get_metadata_snap returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) dm_device_name(pool->pool_md), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) DMEMIT("%llu %llu/%llu %llu/%llu ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) (unsigned long long)transaction_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968) (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) (unsigned long long)nr_blocks_metadata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970) (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) (unsigned long long)nr_blocks_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) if (held_root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) DMEMIT("%llu ", held_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) DMEMIT("- ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) mode = get_pool_mode(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) if (mode == PM_OUT_OF_DATA_SPACE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) DMEMIT("out_of_data_space ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) else if (is_read_only_pool_mode(mode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) DMEMIT("ro ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984) DMEMIT("rw ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) if (!pool->pf.discard_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) DMEMIT("ignore_discard ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) else if (pool->pf.discard_passdown)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) DMEMIT("discard_passdown ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) DMEMIT("no_discard_passdown ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) if (pool->pf.error_if_no_space)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) DMEMIT("error_if_no_space ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996) DMEMIT("queue_if_no_space ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) if (dm_pool_metadata_needs_check(pool->pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) DMEMIT("needs_check ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) DMEMIT("- ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003) DMEMIT("%llu ", (unsigned long long)calc_metadata_threshold(pt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) case STATUSTYPE_TABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) DMEMIT("%s %s %lu %llu ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010) format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) (unsigned long)pool->sectors_per_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) (unsigned long long)pt->low_water_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) emit_flags(&pt->requested_pf, result, sz, maxlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) DMEMIT("Error");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) static int pool_iterate_devices(struct dm_target *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) iterate_devices_callout_fn fn, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027) return fn(ti, pt->data_dev, 0, ti->len, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) struct pool_c *pt = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) struct pool *pool = pt->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037) * If max_sectors is smaller than pool->sectors_per_block adjust it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) * to the highest possible power-of-2 factor of pool->sectors_per_block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) * This is especially beneficial when the pool's data device is a RAID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) * device that has a full stripe width that matches pool->sectors_per_block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041) * -- because even though partial RAID stripe-sized IOs will be issued to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) * single RAID stripe; when aggregated they will end on a full RAID stripe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043) * boundary.. which avoids additional partial RAID stripe writes cascading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) if (limits->max_sectors < pool->sectors_per_block) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) while (!is_factor(pool->sectors_per_block, limits->max_sectors)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) if ((limits->max_sectors & (limits->max_sectors - 1)) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) limits->max_sectors--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) limits->max_sectors = rounddown_pow_of_two(limits->max_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054) * If the system-determined stacked limits are compatible with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) * pool's blocksize (io_opt is a factor) do not override them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) if (io_opt_sectors < pool->sectors_per_block ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) !is_factor(io_opt_sectors, pool->sectors_per_block)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) if (is_factor(pool->sectors_per_block, limits->max_sectors))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) blk_limits_io_min(limits, limits->max_sectors << SECTOR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062) blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067) * pt->adjusted_pf is a staging area for the actual features to use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) * They get transferred to the live pool in bind_control_target()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) * called from pool_preresume().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) if (!pt->adjusted_pf.discard_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) * Must explicitly disallow stacking discard limits otherwise the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) * block layer will stack them if pool's data device has support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) * QUEUE_FLAG_DISCARD wouldn't be set but there is no way for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076) * user to see that, so make sure to set all discard limits to 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) limits->discard_granularity = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082) disable_passdown_if_not_supported(pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) * The pool uses the same discard limits as the underlying data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) * device. DM core has already set this up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) static struct target_type pool_target = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091) .name = "thin-pool",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) DM_TARGET_IMMUTABLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) .version = {1, 22, 0},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) .module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) .ctr = pool_ctr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) .dtr = pool_dtr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) .map = pool_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) .presuspend = pool_presuspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) .presuspend_undo = pool_presuspend_undo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101) .postsuspend = pool_postsuspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) .preresume = pool_preresume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) .resume = pool_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) .message = pool_message,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105) .status = pool_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) .iterate_devices = pool_iterate_devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) .io_hints = pool_io_hints,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) * Thin target methods
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113) static void thin_get(struct thin_c *tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) refcount_inc(&tc->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118) static void thin_put(struct thin_c *tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) if (refcount_dec_and_test(&tc->refcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121) complete(&tc->can_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) static void thin_dtr(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) struct thin_c *tc = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128) spin_lock_irq(&tc->pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) list_del_rcu(&tc->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) spin_unlock_irq(&tc->pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) thin_put(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) wait_for_completion(&tc->can_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) mutex_lock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) __pool_dec(tc->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) dm_pool_close_thin_device(tc->td);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) dm_put_device(ti, tc->pool_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) if (tc->origin_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142) dm_put_device(ti, tc->origin_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) kfree(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145) mutex_unlock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) * Thin target parameters:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151) * <pool_dev> <dev_id> [origin_dev]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154) * dev_id: the internal device identifier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) * origin_dev: a device external to the pool that should act as the origin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) * If the pool device has discards disabled, they get disabled for the thin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158) * device as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160) static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163) struct thin_c *tc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164) struct dm_dev *pool_dev, *origin_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) struct mapped_device *pool_md;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167) mutex_lock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169) if (argc != 2 && argc != 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) ti->error = "Invalid argument count";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) if (!tc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177) ti->error = "Out of memory";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) tc->thin_md = dm_table_get_md(ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) spin_lock_init(&tc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) INIT_LIST_HEAD(&tc->deferred_cells);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184) bio_list_init(&tc->deferred_bio_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) bio_list_init(&tc->retry_on_resume_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186) tc->sort_bio_list = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188) if (argc == 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) if (!strcmp(argv[0], argv[2])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) ti->error = "Error setting origin device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) goto bad_origin_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) r = dm_get_device(ti, argv[2], FMODE_READ, &origin_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) ti->error = "Error opening origin device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) goto bad_origin_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200) tc->origin_dev = origin_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203) r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205) ti->error = "Error opening pool device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) goto bad_pool_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) tc->pool_dev = pool_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) ti->error = "Invalid device id";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) goto bad_common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216) pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) if (!pool_md) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) ti->error = "Couldn't get pool mapped device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220) goto bad_common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223) tc->pool = __pool_table_lookup(pool_md);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) if (!tc->pool) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) ti->error = "Couldn't find pool object";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) goto bad_pool_lookup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) __pool_inc(tc->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231) if (get_pool_mode(tc->pool) == PM_FAIL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232) ti->error = "Couldn't open thin device, Pool is in fail mode";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234) goto bad_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237) r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239) ti->error = "Couldn't open thin internal device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240) goto bad_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) ti->num_flush_bios = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248) ti->flush_supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249) ti->per_io_data_size = sizeof(struct dm_thin_endio_hook);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) /* In case the pool supports discards, pass them on. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252) if (tc->pool->pf.discard_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) ti->discards_supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254) ti->num_discard_bios = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257) mutex_unlock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) spin_lock_irq(&tc->pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260) if (tc->pool->suspended) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261) spin_unlock_irq(&tc->pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263) ti->error = "Unable to activate thin device while pool is suspended";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267) refcount_set(&tc->refcount, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) init_completion(&tc->can_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269) list_add_tail_rcu(&tc->list, &tc->pool->active_thins);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270) spin_unlock_irq(&tc->pool->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272) * This synchronize_rcu() call is needed here otherwise we risk a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) * wake_worker() call finding no bios to process (because the newly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274) * added tc isn't yet visible). So this reduces latency since we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275) * aren't then dependent on the periodic commit to wake_worker().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) dm_put(pool_md);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) bad:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284) dm_pool_close_thin_device(tc->td);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285) bad_pool:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286) __pool_dec(tc->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) bad_pool_lookup:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) dm_put(pool_md);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) bad_common:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290) dm_put_device(ti, tc->pool_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) bad_pool_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292) if (tc->origin_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) dm_put_device(ti, tc->origin_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294) bad_origin_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) kfree(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297) mutex_unlock(&dm_thin_pool_table.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) static int thin_map(struct dm_target *ti, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) return thin_bio_map(ti, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) static int thin_endio(struct dm_target *ti, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) blk_status_t *err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313) struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314) struct list_head work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) struct dm_thin_new_mapping *m, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316) struct pool *pool = h->tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) if (h->shared_read_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319) INIT_LIST_HEAD(&work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320) dm_deferred_entry_dec(h->shared_read_entry, &work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323) list_for_each_entry_safe(m, tmp, &work, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324) list_del(&m->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) __complete_mapping_preparation(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330) if (h->all_io_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331) INIT_LIST_HEAD(&work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332) dm_deferred_entry_dec(h->all_io_entry, &work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333) if (!list_empty(&work)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) spin_lock_irqsave(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335) list_for_each_entry_safe(m, tmp, &work, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) list_add_tail(&m->list, &pool->prepared_discards);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) spin_unlock_irqrestore(&pool->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338) wake_worker(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342) if (h->cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) cell_defer_no_holder(h->tc, h->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) return DM_ENDIO_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348) static void thin_presuspend(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) struct thin_c *tc = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352) if (dm_noflush_suspending(ti))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) noflush_work(tc, do_noflush_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) static void thin_postsuspend(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358) struct thin_c *tc = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361) * The dm_noflush_suspending flag has been cleared by now, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362) * unfortunately we must always run this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) noflush_work(tc, do_noflush_stop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) static int thin_preresume(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) struct thin_c *tc = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371) if (tc->origin_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) tc->origin_size = get_dev_size(tc->origin_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) * <nr mapped sectors> <highest mapped sector>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) static void thin_status(struct dm_target *ti, status_type_t type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381) unsigned status_flags, char *result, unsigned maxlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384) ssize_t sz = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385) dm_block_t mapped, highest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) char buf[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) struct thin_c *tc = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389) if (get_pool_mode(tc->pool) == PM_FAIL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390) DMEMIT("Fail");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) if (!tc->td)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395) DMEMIT("-");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398) case STATUSTYPE_INFO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399) r = dm_thin_get_mapped_count(tc->td, &mapped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401) DMERR("dm_thin_get_mapped_count returned %d", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405) r = dm_thin_get_highest_mapped_block(tc->td, &highest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) if (r < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407) DMERR("dm_thin_get_highest_mapped_block returned %d", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) DMEMIT("%llu", ((highest + 1) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) tc->pool->sectors_per_block) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416) DMEMIT("-");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) case STATUSTYPE_TABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420) DMEMIT("%s %lu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422) (unsigned long) tc->dev_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423) if (tc->origin_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) DMEMIT("Error");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435) static int thin_iterate_devices(struct dm_target *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) iterate_devices_callout_fn fn, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) sector_t blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439) struct thin_c *tc = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) * We can't call dm_pool_get_data_dev_size() since that blocks. So
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444) * we follow a more convoluted path through to the pool's target.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446) if (!pool->ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) return 0; /* nothing is bound */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449) blocks = pool->ti->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450) (void) sector_div(blocks, pool->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451) if (blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452) return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) struct thin_c *tc = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) struct pool *pool = tc->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462) if (!pool->pf.discard_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465) limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) limits->max_discard_sectors = 2048 * 1024 * 16; /* 16G */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) static struct target_type thin_target = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) .name = "thin",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) .version = {1, 22, 0},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472) .module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473) .ctr = thin_ctr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474) .dtr = thin_dtr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) .map = thin_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476) .end_io = thin_endio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477) .preresume = thin_preresume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478) .presuspend = thin_presuspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479) .postsuspend = thin_postsuspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) .status = thin_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) .iterate_devices = thin_iterate_devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482) .io_hints = thin_io_hints,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487) static int __init dm_thin_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) int r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491) pool_table_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) _new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494) if (!_new_mapping_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497) r = dm_register_target(&thin_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) goto bad_new_mapping_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501) r = dm_register_target(&pool_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) goto bad_thin_target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) bad_thin_target:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) dm_unregister_target(&thin_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509) bad_new_mapping_cache:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) kmem_cache_destroy(_new_mapping_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515) static void dm_thin_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517) dm_unregister_target(&thin_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) dm_unregister_target(&pool_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) kmem_cache_destroy(_new_mapping_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) pool_table_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525) module_init(dm_thin_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526) module_exit(dm_thin_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528) module_param_named(no_space_timeout, no_space_timeout_secs, uint, S_IRUGO | S_IWUSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531) MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532) MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) MODULE_LICENSE("GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags