^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2019 Arrikto, Inc. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/bio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/log2.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/dm-io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/kdev_t.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/blk_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/dm-kcopyd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/backing-dev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/device-mapper.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include "dm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include "dm-clone-metadata.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define DM_MSG_PREFIX "clone"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * Minimum and maximum allowed region sizes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define MIN_REGION_SIZE (1 << 3) /* 4KB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define MAX_REGION_SIZE (1 << 21) /* 1GB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define MIN_HYDRATIONS 256 /* Size of hydration mempool */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define DEFAULT_HYDRATION_THRESHOLD 1 /* 1 region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define DEFAULT_HYDRATION_BATCH_SIZE 1 /* Hydrate in batches of 1 region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define COMMIT_PERIOD HZ /* 1 sec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * Hydration hash table size: 1 << HASH_TABLE_BITS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define HASH_TABLE_BITS 15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(clone_hydration_throttle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) "A percentage of time allocated for hydrating regions");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) /* Slab cache for struct dm_clone_region_hydration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static struct kmem_cache *_hydration_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /* dm-clone metadata modes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) enum clone_metadata_mode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) CM_WRITE, /* metadata may be changed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) CM_READ_ONLY, /* metadata may not be changed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) CM_FAIL, /* all metadata I/O fails */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct hash_table_bucket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct clone {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct dm_target *ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) struct dm_dev *metadata_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct dm_dev *dest_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct dm_dev *source_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) unsigned long nr_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) sector_t region_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) unsigned int region_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * A metadata commit and the actions taken in case it fails should run
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * as a single atomic step.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) struct mutex commit_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) struct dm_clone_metadata *cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * bio used to flush the destination device, before committing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct bio flush_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) /* Region hydration hash table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct hash_table_bucket *ht;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) atomic_t ios_in_flight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) wait_queue_head_t hydration_stopped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) mempool_t hydration_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) unsigned long last_commit_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * We defer incoming WRITE bios for regions that are not hydrated,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * until after these regions have been hydrated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * Also, we defer REQ_FUA and REQ_PREFLUSH bios, until after the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * metadata have been committed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) struct bio_list deferred_bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) struct bio_list deferred_discard_bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct bio_list deferred_flush_bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct bio_list deferred_flush_completions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) /* Maximum number of regions being copied during background hydration. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) unsigned int hydration_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) /* Number of regions to batch together during background hydration. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) unsigned int hydration_batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /* Which region to hydrate next */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) unsigned long hydration_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) atomic_t hydrations_in_flight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * Save a copy of the table line rather than reconstructing it for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * status.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) unsigned int nr_ctr_args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) const char **ctr_args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct workqueue_struct *wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct work_struct worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) struct delayed_work waker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct dm_kcopyd_client *kcopyd_client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) enum clone_metadata_mode mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * dm-clone flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) #define DM_CLONE_DISCARD_PASSDOWN 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #define DM_CLONE_HYDRATION_ENABLED 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #define DM_CLONE_HYDRATION_SUSPENDED 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * Metadata failure handling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) static enum clone_metadata_mode get_clone_mode(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) return READ_ONCE(clone->mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static const char *clone_device_name(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return dm_table_device_name(clone->ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static void __set_clone_mode(struct clone *clone, enum clone_metadata_mode new_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) const char *descs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) "read-write",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) "read-only",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) "fail"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) enum clone_metadata_mode old_mode = get_clone_mode(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /* Never move out of fail mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (old_mode == CM_FAIL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) new_mode = CM_FAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) switch (new_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) case CM_FAIL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) case CM_READ_ONLY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) dm_clone_metadata_set_read_only(clone->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) case CM_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) dm_clone_metadata_set_read_write(clone->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) WRITE_ONCE(clone->mode, new_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (new_mode != old_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) dm_table_event(clone->ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) DMINFO("%s: Switching to %s mode", clone_device_name(clone),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) descs[(int)new_mode]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) static void __abort_transaction(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) const char *dev_name = clone_device_name(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (get_clone_mode(clone) >= CM_READ_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) DMERR("%s: Aborting current metadata transaction", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (dm_clone_metadata_abort(clone->cmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) DMERR("%s: Failed to abort metadata transaction", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) __set_clone_mode(clone, CM_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static void __reload_in_core_bitset(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) const char *dev_name = clone_device_name(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (get_clone_mode(clone) == CM_FAIL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) /* Reload the on-disk bitset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) DMINFO("%s: Reloading on-disk bitmap", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (dm_clone_reload_in_core_bitset(clone->cmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) DMERR("%s: Failed to reload on-disk bitmap", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) __set_clone_mode(clone, CM_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) static void __metadata_operation_failed(struct clone *clone, const char *op, int r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) DMERR("%s: Metadata operation `%s' failed: error = %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) clone_device_name(clone), op, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) __abort_transaction(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) __set_clone_mode(clone, CM_READ_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * dm_clone_reload_in_core_bitset() may run concurrently with either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * dm_clone_set_region_hydrated() or dm_clone_cond_set_range(), but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * it's safe as we have already set the metadata to read-only mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) __reload_in_core_bitset(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /* Wake up anyone waiting for region hydrations to stop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) static inline void wakeup_hydration_waiters(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) wake_up_all(&clone->hydration_stopped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) static inline void wake_worker(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) queue_work(clone->wq, &clone->worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * bio helper functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) static inline void remap_to_source(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) bio_set_dev(bio, clone->source_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) static inline void remap_to_dest(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) bio_set_dev(bio, clone->dest_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) static bool bio_triggers_commit(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return op_is_flush(bio->bi_opf) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) dm_clone_changed_this_transaction(clone->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) /* Get the address of the region in sectors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) static inline sector_t region_to_sector(struct clone *clone, unsigned long region_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return ((sector_t)region_nr << clone->region_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) /* Get the region number of the bio */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static inline unsigned long bio_to_region(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return (bio->bi_iter.bi_sector >> clone->region_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /* Get the region range covered by the bio */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static void bio_region_range(struct clone *clone, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) unsigned long *rs, unsigned long *nr_regions)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) unsigned long end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) *rs = dm_sector_div_up(bio->bi_iter.bi_sector, clone->region_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) end = bio_end_sector(bio) >> clone->region_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (*rs >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) *nr_regions = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) *nr_regions = end - *rs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* Check whether a bio overwrites a region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) static inline bool is_overwrite_bio(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) return (bio_data_dir(bio) == WRITE && bio_sectors(bio) == clone->region_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) static void fail_bios(struct bio_list *bios, blk_status_t status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) while ((bio = bio_list_pop(bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) bio->bi_status = status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) static void submit_bios(struct bio_list *bios)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) while ((bio = bio_list_pop(bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * Submit bio to the underlying device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * If the bio triggers a commit, delay it, until after the metadata have been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * committed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * NOTE: The bio remapping must be performed by the caller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) static void issue_bio(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) if (!bio_triggers_commit(clone, bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) * If the metadata mode is RO or FAIL we won't be able to commit the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * metadata, so we complete the bio with an error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * Batch together any bios that trigger commits and then issue a single
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * commit for them in process_deferred_flush_bios().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) spin_lock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) bio_list_add(&clone->deferred_flush_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) spin_unlock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) wake_worker(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * Remap bio to the destination device and submit it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * If the bio triggers a commit, delay it, until after the metadata have been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * committed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) static void remap_and_issue(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) remap_to_dest(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) issue_bio(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * Issue bios that have been deferred until after their region has finished
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * hydrating.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * We delegate the bio submission to the worker thread, so this is safe to call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * from interrupt context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) static void issue_deferred_bios(struct clone *clone, struct bio_list *bios)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) struct bio_list flush_bios = BIO_EMPTY_LIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) struct bio_list normal_bios = BIO_EMPTY_LIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (bio_list_empty(bios))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) while ((bio = bio_list_pop(bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) if (bio_triggers_commit(clone, bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) bio_list_add(&flush_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) bio_list_add(&normal_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) spin_lock_irqsave(&clone->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) bio_list_merge(&clone->deferred_bios, &normal_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) bio_list_merge(&clone->deferred_flush_bios, &flush_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) spin_unlock_irqrestore(&clone->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) wake_worker(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) static void complete_overwrite_bio(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * If the bio has the REQ_FUA flag set we must commit the metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * before signaling its completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * complete_overwrite_bio() is only called by hydration_complete(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * after having successfully updated the metadata. This means we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * need to call dm_clone_changed_this_transaction() to check if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * metadata has changed and thus we can avoid taking the metadata spin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) if (!(bio->bi_opf & REQ_FUA)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * If the metadata mode is RO or FAIL we won't be able to commit the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * metadata, so we complete the bio with an error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * Batch together any bios that trigger commits and then issue a single
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * commit for them in process_deferred_flush_bios().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) spin_lock_irqsave(&clone->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) bio_list_add(&clone->deferred_flush_completions, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) spin_unlock_irqrestore(&clone->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) wake_worker(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) static void trim_bio(struct bio *bio, sector_t sector, unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) bio->bi_iter.bi_sector = sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) bio->bi_iter.bi_size = to_bytes(len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) static void complete_discard_bio(struct clone *clone, struct bio *bio, bool success)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) unsigned long rs, nr_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * If the destination device supports discards, remap and trim the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * discard bio and pass it down. Otherwise complete the bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * immediately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags) && success) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) remap_to_dest(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) bio_region_range(clone, bio, &rs, &nr_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) trim_bio(bio, region_to_sector(clone, rs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) nr_regions << clone->region_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) static void process_discard_bio(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) unsigned long rs, nr_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) bio_region_range(clone, bio, &rs, &nr_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (!nr_regions) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (WARN_ON(rs >= clone->nr_regions || (rs + nr_regions) < rs ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) (rs + nr_regions) > clone->nr_regions)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) DMERR("%s: Invalid range (%lu + %lu, total regions %lu) for discard (%llu + %u)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) clone_device_name(clone), rs, nr_regions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) clone->nr_regions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) (unsigned long long)bio->bi_iter.bi_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) bio_sectors(bio));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * The covered regions are already hydrated so we just need to pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * down the discard.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (dm_clone_is_range_hydrated(clone->cmd, rs, nr_regions)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) complete_discard_bio(clone, bio, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * If the metadata mode is RO or FAIL we won't be able to update the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * metadata for the regions covered by the discard so we just ignore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * Defer discard processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) spin_lock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) bio_list_add(&clone->deferred_discard_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) spin_unlock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) wake_worker(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * dm-clone region hydrations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct dm_clone_region_hydration {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) struct clone *clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) unsigned long region_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) struct bio *overwrite_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) bio_end_io_t *overwrite_bio_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) struct bio_list deferred_bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) blk_status_t status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) /* Used by hydration batching */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) /* Used by hydration hash table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) struct hlist_node h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * Hydration hash table implementation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * Ideally we would like to use list_bl, which uses bit spin locks and employs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * the least significant bit of the list head to lock the corresponding bucket,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * reducing the memory overhead for the locks. But, currently, list_bl and bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * spin locks don't support IRQ safe versions. Since we have to take the lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * in both process and interrupt context, we must fall back to using regular
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * spin locks; one per hash table bucket.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) struct hash_table_bucket {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct hlist_head head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) /* Spinlock protecting the bucket */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) #define bucket_lock_irqsave(bucket, flags) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) spin_lock_irqsave(&(bucket)->lock, flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) #define bucket_unlock_irqrestore(bucket, flags) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) spin_unlock_irqrestore(&(bucket)->lock, flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) #define bucket_lock_irq(bucket) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) spin_lock_irq(&(bucket)->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) #define bucket_unlock_irq(bucket) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) spin_unlock_irq(&(bucket)->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) static int hash_table_init(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) unsigned int i, sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) struct hash_table_bucket *bucket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) sz = 1 << HASH_TABLE_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) clone->ht = kvmalloc(sz * sizeof(struct hash_table_bucket), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (!clone->ht)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) for (i = 0; i < sz; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) bucket = clone->ht + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) INIT_HLIST_HEAD(&bucket->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) spin_lock_init(&bucket->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) static void hash_table_exit(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) kvfree(clone->ht);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) static struct hash_table_bucket *get_hash_table_bucket(struct clone *clone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) unsigned long region_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) return &clone->ht[hash_long(region_nr, HASH_TABLE_BITS)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) * Search hash table for a hydration with hd->region_nr == region_nr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * NOTE: Must be called with the bucket lock held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) static struct dm_clone_region_hydration *__hash_find(struct hash_table_bucket *bucket,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) unsigned long region_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct dm_clone_region_hydration *hd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) hlist_for_each_entry(hd, &bucket->head, h) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) if (hd->region_nr == region_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) return hd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * Insert a hydration into the hash table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * NOTE: Must be called with the bucket lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) static inline void __insert_region_hydration(struct hash_table_bucket *bucket,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) struct dm_clone_region_hydration *hd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) hlist_add_head(&hd->h, &bucket->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * This function inserts a hydration into the hash table, unless someone else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * managed to insert a hydration for the same region first. In the latter case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * it returns the existing hydration descriptor for this region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * NOTE: Must be called with the hydration hash table lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) static struct dm_clone_region_hydration *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) __find_or_insert_region_hydration(struct hash_table_bucket *bucket,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) struct dm_clone_region_hydration *hd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) struct dm_clone_region_hydration *hd2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) hd2 = __hash_find(bucket, hd->region_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) if (hd2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) return hd2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) __insert_region_hydration(bucket, hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) return hd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) /* Allocate a hydration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) static struct dm_clone_region_hydration *alloc_hydration(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) struct dm_clone_region_hydration *hd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * Allocate a hydration from the hydration mempool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * This might block but it can't fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) hd = mempool_alloc(&clone->hydration_pool, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) hd->clone = clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) return hd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) static inline void free_hydration(struct dm_clone_region_hydration *hd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) mempool_free(hd, &hd->clone->hydration_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) /* Initialize a hydration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) static void hydration_init(struct dm_clone_region_hydration *hd, unsigned long region_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) hd->region_nr = region_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) hd->overwrite_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) bio_list_init(&hd->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) hd->status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) INIT_LIST_HEAD(&hd->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) INIT_HLIST_NODE(&hd->h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * Update dm-clone's metadata after a region has finished hydrating and remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * hydration from the hash table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) static int hydration_update_metadata(struct dm_clone_region_hydration *hd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) struct hash_table_bucket *bucket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) struct clone *clone = hd->clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) r = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) /* Update the metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if (likely(!r) && hd->status == BLK_STS_OK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) r = dm_clone_set_region_hydrated(clone->cmd, hd->region_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) bucket = get_hash_table_bucket(clone, hd->region_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) /* Remove hydration from hash table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) bucket_lock_irqsave(bucket, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) hlist_del(&hd->h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) bucket_unlock_irqrestore(bucket, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) * Complete a region's hydration:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) * 1. Update dm-clone's metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) * 2. Remove hydration from hash table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) * 3. Complete overwrite bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) * 4. Issue deferred bios.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) * 5. If this was the last hydration, wake up anyone waiting for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) * hydrations to finish.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) static void hydration_complete(struct dm_clone_region_hydration *hd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) blk_status_t status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) struct clone *clone = hd->clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) r = hydration_update_metadata(hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (hd->status == BLK_STS_OK && likely(!r)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) if (hd->overwrite_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) complete_overwrite_bio(clone, hd->overwrite_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) issue_deferred_bios(clone, &hd->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) status = r ? BLK_STS_IOERR : hd->status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (hd->overwrite_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) bio_list_add(&hd->deferred_bios, hd->overwrite_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) fail_bios(&hd->deferred_bios, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) free_hydration(hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (atomic_dec_and_test(&clone->hydrations_in_flight))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) wakeup_hydration_waiters(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) static void hydration_kcopyd_callback(int read_err, unsigned long write_err, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) blk_status_t status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) struct dm_clone_region_hydration *tmp, *hd = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) struct clone *clone = hd->clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) LIST_HEAD(batched_hydrations);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) if (read_err || write_err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) DMERR_LIMIT("%s: hydration failed", clone_device_name(clone));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) status = BLK_STS_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) list_splice_tail(&hd->list, &batched_hydrations);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) hd->status = status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) hydration_complete(hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) /* Complete batched hydrations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) list_for_each_entry_safe(hd, tmp, &batched_hydrations, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) hd->status = status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) hydration_complete(hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) /* Continue background hydration, if there is no I/O in-flight */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) if (test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) !atomic_read(&clone->ios_in_flight))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) wake_worker(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) static void hydration_copy(struct dm_clone_region_hydration *hd, unsigned int nr_regions)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) unsigned long region_start, region_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) sector_t tail_size, region_size, total_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) struct dm_io_region from, to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) struct clone *clone = hd->clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) if (WARN_ON(!nr_regions))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) region_size = clone->region_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) region_start = hd->region_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) region_end = region_start + nr_regions - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) total_size = region_to_sector(clone, nr_regions - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) if (region_end == clone->nr_regions - 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * The last region of the target might be smaller than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) * region_size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) tail_size = clone->ti->len & (region_size - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (!tail_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) tail_size = region_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) tail_size = region_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) total_size += tail_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) from.bdev = clone->source_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) from.sector = region_to_sector(clone, region_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) from.count = total_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) to.bdev = clone->dest_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) to.sector = from.sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) to.count = from.count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) /* Issue copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) atomic_add(nr_regions, &clone->hydrations_in_flight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) dm_kcopyd_copy(clone->kcopyd_client, &from, 1, &to, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) hydration_kcopyd_callback, hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) static void overwrite_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) struct dm_clone_region_hydration *hd = bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) bio->bi_end_io = hd->overwrite_bio_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) hd->status = bio->bi_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) hydration_complete(hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) static void hydration_overwrite(struct dm_clone_region_hydration *hd, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) * We don't need to save and restore bio->bi_private because device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) * mapper core generates a new bio for us to use, with clean
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) * bi_private.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) hd->overwrite_bio = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) hd->overwrite_bio_end_io = bio->bi_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) bio->bi_end_io = overwrite_endio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) bio->bi_private = hd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) atomic_inc(&hd->clone->hydrations_in_flight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * Hydrate bio's region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) * This function starts the hydration of the bio's region and puts the bio in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * the list of deferred bios for this region. In case, by the time this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * function is called, the region has finished hydrating it's submitted to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * destination device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * NOTE: The bio remapping must be performed by the caller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) static void hydrate_bio_region(struct clone *clone, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) unsigned long region_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) struct hash_table_bucket *bucket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) struct dm_clone_region_hydration *hd, *hd2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) region_nr = bio_to_region(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) bucket = get_hash_table_bucket(clone, region_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) bucket_lock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) hd = __hash_find(bucket, region_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) if (hd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) /* Someone else is hydrating the region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) bio_list_add(&hd->deferred_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) /* The region has been hydrated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) issue_bio(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) * We must allocate a hydration descriptor and start the hydration of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) * the corresponding region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) hd = alloc_hydration(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) hydration_init(hd, region_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) bucket_lock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) /* Check if the region has been hydrated in the meantime. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) free_hydration(hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) issue_bio(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) hd2 = __find_or_insert_region_hydration(bucket, hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) if (hd2 != hd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) /* Someone else started the region's hydration. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) bio_list_add(&hd2->deferred_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) free_hydration(hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) * If the metadata mode is RO or FAIL then there is no point starting a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) * hydration, since we will not be able to update the metadata when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) * hydration finishes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) hlist_del(&hd->h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) free_hydration(hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) * Start region hydration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * If a bio overwrites a region, i.e., its size is equal to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) * region's size, then we don't need to copy the region from the source
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * to the destination device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) if (is_overwrite_bio(clone, bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) hydration_overwrite(hd, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) bio_list_add(&hd->deferred_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) hydration_copy(hd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) * Background hydrations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) * Batch region hydrations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) * To better utilize device bandwidth we batch together the hydration of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) * adjacent regions. This allows us to use small region sizes, e.g., 4KB, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) * is good for small, random write performance (because of the overwriting of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) * un-hydrated regions) and at the same time issue big copy requests to kcopyd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * to achieve high hydration bandwidth.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) struct batch_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) struct dm_clone_region_hydration *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) unsigned int nr_batched_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) static void __batch_hydration(struct batch_info *batch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) struct dm_clone_region_hydration *hd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) struct clone *clone = hd->clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) unsigned int max_batch_size = READ_ONCE(clone->hydration_batch_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) if (batch->head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /* Try to extend the current batch */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) if (batch->nr_batched_regions < max_batch_size &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) (batch->head->region_nr + batch->nr_batched_regions) == hd->region_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) list_add_tail(&hd->list, &batch->head->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) batch->nr_batched_regions++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) hd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) /* Check if we should issue the current batch */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) if (batch->nr_batched_regions >= max_batch_size || hd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) hydration_copy(batch->head, batch->nr_batched_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) batch->head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) batch->nr_batched_regions = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) if (!hd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) /* We treat max batch sizes of zero and one equivalently */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) if (max_batch_size <= 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) hydration_copy(hd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) /* Start a new batch */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) BUG_ON(!list_empty(&hd->list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) batch->head = hd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) batch->nr_batched_regions = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) static unsigned long __start_next_hydration(struct clone *clone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) unsigned long offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) struct batch_info *batch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) struct hash_table_bucket *bucket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) struct dm_clone_region_hydration *hd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) unsigned long nr_regions = clone->nr_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) hd = alloc_hydration(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) /* Try to find a region to hydrate. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) offset = dm_clone_find_next_unhydrated_region(clone->cmd, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) if (offset == nr_regions)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) bucket = get_hash_table_bucket(clone, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) bucket_lock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) if (!dm_clone_is_region_hydrated(clone->cmd, offset) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) !__hash_find(bucket, offset)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) hydration_init(hd, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) __insert_region_hydration(bucket, hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) /* Batch hydration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) __batch_hydration(batch, hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) return (offset + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) bucket_unlock_irq(bucket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) } while (++offset < nr_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (hd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) free_hydration(hd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) return offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) * This function searches for regions that still reside in the source device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) * and starts their hydration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) static void do_hydration(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) unsigned int current_volume;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) unsigned long offset, nr_regions = clone->nr_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) struct batch_info batch = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) .head = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) .nr_batched_regions = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if (dm_clone_is_hydration_done(clone->cmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * Avoid race with device suspension.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) atomic_inc(&clone->hydrations_in_flight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * Make sure atomic_inc() is ordered before test_bit(), otherwise we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) * might race with clone_postsuspend() and start a region hydration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * after the target has been suspended.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) * This is paired with the smp_mb__after_atomic() in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) * clone_postsuspend().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) offset = clone->hydration_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) while (likely(!test_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) !atomic_read(&clone->ios_in_flight) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) offset < nr_regions) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) current_volume = atomic_read(&clone->hydrations_in_flight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) current_volume += batch.nr_batched_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) if (current_volume > READ_ONCE(clone->hydration_threshold))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) offset = __start_next_hydration(clone, offset, &batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) if (batch.head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) hydration_copy(batch.head, batch.nr_batched_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) if (offset >= nr_regions)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) clone->hydration_offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) if (atomic_dec_and_test(&clone->hydrations_in_flight))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) wakeup_hydration_waiters(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) static bool need_commit_due_to_time(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) return !time_in_range(jiffies, clone->last_commit_jiffies,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) clone->last_commit_jiffies + COMMIT_PERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) * A non-zero return indicates read-only or fail mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) static int commit_metadata(struct clone *clone, bool *dest_dev_flushed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) if (dest_dev_flushed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) *dest_dev_flushed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) mutex_lock(&clone->commit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) if (!dm_clone_changed_this_transaction(clone->cmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) r = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) r = dm_clone_metadata_pre_commit(clone->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) if (unlikely(r)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) __metadata_operation_failed(clone, "dm_clone_metadata_pre_commit", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) bio_reset(&clone->flush_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) bio_set_dev(&clone->flush_bio, clone->dest_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) clone->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) r = submit_bio_wait(&clone->flush_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) if (unlikely(r)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) __metadata_operation_failed(clone, "flush destination device", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) if (dest_dev_flushed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) *dest_dev_flushed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) r = dm_clone_metadata_commit(clone->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) if (unlikely(r)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) __metadata_operation_failed(clone, "dm_clone_metadata_commit", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) if (dm_clone_is_hydration_done(clone->cmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) dm_table_event(clone->ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) mutex_unlock(&clone->commit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) static void process_deferred_discards(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) int r = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) unsigned long rs, nr_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) struct bio_list discards = BIO_EMPTY_LIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) spin_lock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) bio_list_merge(&discards, &clone->deferred_discard_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) bio_list_init(&clone->deferred_discard_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) spin_unlock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) if (bio_list_empty(&discards))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) /* Update the metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) bio_list_for_each(bio, &discards) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) bio_region_range(clone, bio, &rs, &nr_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) * A discard request might cover regions that have been already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) * hydrated. There is no need to update the metadata for these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) * regions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) r = dm_clone_cond_set_range(clone->cmd, rs, nr_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (unlikely(r))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) while ((bio = bio_list_pop(&discards)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) complete_discard_bio(clone, bio, r == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) static void process_deferred_bios(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) struct bio_list bios = BIO_EMPTY_LIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) spin_lock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) bio_list_merge(&bios, &clone->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) bio_list_init(&clone->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) spin_unlock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) if (bio_list_empty(&bios))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) submit_bios(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) static void process_deferred_flush_bios(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) bool dest_dev_flushed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) struct bio_list bios = BIO_EMPTY_LIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) struct bio_list bio_completions = BIO_EMPTY_LIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) * If there are any deferred flush bios, we must commit the metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) * before issuing them or signaling their completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) spin_lock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) bio_list_merge(&bios, &clone->deferred_flush_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) bio_list_init(&clone->deferred_flush_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) bio_list_merge(&bio_completions, &clone->deferred_flush_completions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) bio_list_init(&clone->deferred_flush_completions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) spin_unlock_irq(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) !(dm_clone_changed_this_transaction(clone->cmd) && need_commit_due_to_time(clone)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) if (commit_metadata(clone, &dest_dev_flushed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) bio_list_merge(&bios, &bio_completions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) while ((bio = bio_list_pop(&bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) clone->last_commit_jiffies = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) while ((bio = bio_list_pop(&bio_completions)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) while ((bio = bio_list_pop(&bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if ((bio->bi_opf & REQ_PREFLUSH) && dest_dev_flushed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) /* We just flushed the destination device as part of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) * the metadata commit, so there is no reason to send
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) * another flush.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) static void do_worker(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) struct clone *clone = container_of(work, typeof(*clone), worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) process_deferred_bios(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) process_deferred_discards(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * process_deferred_flush_bios():
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) * - Commit metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) * - Process deferred REQ_FUA completions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) * - Process deferred REQ_PREFLUSH bios
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) process_deferred_flush_bios(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) /* Background hydration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) do_hydration(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) * Commit periodically so that not too much unwritten data builds up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) * Also, restart background hydration, if it has been stopped by in-flight I/O.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) static void do_waker(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) struct clone *clone = container_of(to_delayed_work(work), struct clone, waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) wake_worker(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) queue_delayed_work(clone->wq, &clone->waker, COMMIT_PERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) * Target methods
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) static int clone_map(struct dm_target *ti, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) struct clone *clone = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) unsigned long region_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) atomic_inc(&clone->ios_in_flight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) if (unlikely(get_clone_mode(clone) == CM_FAIL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) return DM_MAPIO_KILL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) * REQ_PREFLUSH bios carry no data:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) * - Commit metadata, if changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) * - Pass down to destination device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) if (bio->bi_opf & REQ_PREFLUSH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) remap_and_issue(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) * dm-clone interprets discards and performs a fast hydration of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) * discarded regions, i.e., we skip the copy from the source device and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) * just mark the regions as hydrated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) if (bio_op(bio) == REQ_OP_DISCARD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) process_discard_bio(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) * If the bio's region is hydrated, redirect it to the destination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) * device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * If the region is not hydrated and the bio is a READ, redirect it to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) * the source device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) * Else, defer WRITE bio until after its region has been hydrated and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) * start the region's hydration immediately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) region_nr = bio_to_region(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) remap_and_issue(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) } else if (bio_data_dir(bio) == READ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) remap_to_source(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) return DM_MAPIO_REMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) remap_to_dest(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) hydrate_bio_region(clone, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) static int clone_endio(struct dm_target *ti, struct bio *bio, blk_status_t *error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) struct clone *clone = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) atomic_dec(&clone->ios_in_flight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) return DM_ENDIO_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) static void emit_flags(struct clone *clone, char *result, unsigned int maxlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) ssize_t *sz_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) ssize_t sz = *sz_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) unsigned int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) count = !test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) count += !test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) DMEMIT("%u ", count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (!test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) DMEMIT("no_hydration ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) DMEMIT("no_discard_passdown ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) *sz_ptr = sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) static void emit_core_args(struct clone *clone, char *result,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) unsigned int maxlen, ssize_t *sz_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) ssize_t sz = *sz_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) unsigned int count = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) DMEMIT("%u hydration_threshold %u hydration_batch_size %u ", count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) READ_ONCE(clone->hydration_threshold),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) READ_ONCE(clone->hydration_batch_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) *sz_ptr = sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) * Status format:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) * <clone region size> <#hydrated regions>/<#total regions> <#hydrating regions>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) * <#features> <features>* <#core args> <core args>* <clone metadata mode>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) static void clone_status(struct dm_target *ti, status_type_t type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) unsigned int status_flags, char *result,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) unsigned int maxlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) ssize_t sz = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) dm_block_t nr_free_metadata_blocks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) dm_block_t nr_metadata_blocks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) char buf[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) struct clone *clone = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) case STATUSTYPE_INFO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) if (get_clone_mode(clone) == CM_FAIL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) DMEMIT("Fail");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) /* Commit to ensure statistics aren't out-of-date */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) (void) commit_metadata(clone, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) r = dm_clone_get_free_metadata_block_count(clone->cmd, &nr_free_metadata_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) DMERR("%s: dm_clone_get_free_metadata_block_count returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) clone_device_name(clone), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) r = dm_clone_get_metadata_dev_size(clone->cmd, &nr_metadata_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) DMERR("%s: dm_clone_get_metadata_dev_size returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) clone_device_name(clone), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) DMEMIT("%u %llu/%llu %llu %u/%lu %u ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) DM_CLONE_METADATA_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) (unsigned long long)(nr_metadata_blocks - nr_free_metadata_blocks),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) (unsigned long long)nr_metadata_blocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) (unsigned long long)clone->region_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) dm_clone_nr_of_hydrated_regions(clone->cmd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) clone->nr_regions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) atomic_read(&clone->hydrations_in_flight));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) emit_flags(clone, result, maxlen, &sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) emit_core_args(clone, result, maxlen, &sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) switch (get_clone_mode(clone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) case CM_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) DMEMIT("rw");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) case CM_READ_ONLY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) DMEMIT("ro");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) case CM_FAIL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) DMEMIT("Fail");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) case STATUSTYPE_TABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) format_dev_t(buf, clone->metadata_dev->bdev->bd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) DMEMIT("%s ", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) format_dev_t(buf, clone->dest_dev->bdev->bd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) DMEMIT("%s ", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) format_dev_t(buf, clone->source_dev->bdev->bd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) DMEMIT("%s", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) for (i = 0; i < clone->nr_ctr_args; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) DMEMIT(" %s", clone->ctr_args[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) DMEMIT("Error");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) static sector_t get_dev_size(struct dm_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) * Construct a clone device mapping:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) * clone <metadata dev> <destination dev> <source dev> <region size>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) * [<#feature args> [<feature arg>]* [<#core args> [key value]*]]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) * metadata dev: Fast device holding the persistent metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) * destination dev: The destination device, which will become a clone of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) * source device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) * source dev: The read-only source device that gets cloned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) * region size: dm-clone unit size in sectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) * #feature args: Number of feature arguments passed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) * feature args: E.g. no_hydration, no_discard_passdown
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) * #core arguments: An even number of core arguments
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) * core arguments: Key/value pairs for tuning the core
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) * E.g. 'hydration_threshold 256'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) static int parse_feature_args(struct dm_arg_set *as, struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) unsigned int argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) const char *arg_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) struct dm_target *ti = clone->ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) const struct dm_arg args = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) .min = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) .max = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) .error = "Invalid number of feature arguments"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) /* No feature arguments supplied */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) if (!as->argc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) r = dm_read_arg_group(&args, as, &argc, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) while (argc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) arg_name = dm_shift_arg(as);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) argc--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) if (!strcasecmp(arg_name, "no_hydration")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) __clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) } else if (!strcasecmp(arg_name, "no_discard_passdown")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) __clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) ti->error = "Invalid feature argument";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) static int parse_core_args(struct dm_arg_set *as, struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) unsigned int argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) unsigned int value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) const char *arg_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) struct dm_target *ti = clone->ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) const struct dm_arg args = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) .min = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) .max = 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) .error = "Invalid number of core arguments"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) /* Initialize core arguments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) clone->hydration_batch_size = DEFAULT_HYDRATION_BATCH_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) clone->hydration_threshold = DEFAULT_HYDRATION_THRESHOLD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) /* No core arguments supplied */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) if (!as->argc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) r = dm_read_arg_group(&args, as, &argc, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) if (argc & 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) ti->error = "Number of core arguments must be even";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) while (argc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) arg_name = dm_shift_arg(as);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) argc -= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) if (!strcasecmp(arg_name, "hydration_threshold")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) if (kstrtouint(dm_shift_arg(as), 10, &value)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) ti->error = "Invalid value for argument `hydration_threshold'";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) clone->hydration_threshold = value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) } else if (!strcasecmp(arg_name, "hydration_batch_size")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) if (kstrtouint(dm_shift_arg(as), 10, &value)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) ti->error = "Invalid value for argument `hydration_batch_size'";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) clone->hydration_batch_size = value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) ti->error = "Invalid core argument";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) static int parse_region_size(struct clone *clone, struct dm_arg_set *as, char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) unsigned int region_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) struct dm_arg arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) arg.min = MIN_REGION_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) arg.max = MAX_REGION_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) arg.error = "Invalid region size";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) r = dm_read_arg(&arg, as, ®ion_size, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) /* Check region size is a power of 2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) if (!is_power_of_2(region_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) *error = "Region size is not a power of 2";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) /* Validate the region size against the device logical block size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) if (region_size % (bdev_logical_block_size(clone->source_dev->bdev) >> 9) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) region_size % (bdev_logical_block_size(clone->dest_dev->bdev) >> 9)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) *error = "Region size is not a multiple of device logical block size";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) clone->region_size = region_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) static int validate_nr_regions(unsigned long n, char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) * dm_bitset restricts us to 2^32 regions. test_bit & co. restrict us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) * further to 2^31 regions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) if (n > (1UL << 31)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) *error = "Too many regions. Consider increasing the region size";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) static int parse_metadata_dev(struct clone *clone, struct dm_arg_set *as, char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) sector_t metadata_dev_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) &clone->metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) *error = "Error opening metadata device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) metadata_dev_size = get_dev_size(clone->metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) if (metadata_dev_size > DM_CLONE_METADATA_MAX_SECTORS_WARNING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) bdevname(clone->metadata_dev->bdev, b), DM_CLONE_METADATA_MAX_SECTORS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) static int parse_dest_dev(struct clone *clone, struct dm_arg_set *as, char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) sector_t dest_dev_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) &clone->dest_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) *error = "Error opening destination device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) dest_dev_size = get_dev_size(clone->dest_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) if (dest_dev_size < clone->ti->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) dm_put_device(clone->ti, clone->dest_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) *error = "Device size larger than destination device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) static int parse_source_dev(struct clone *clone, struct dm_arg_set *as, char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) sector_t source_dev_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) &clone->source_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) *error = "Error opening source device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) source_dev_size = get_dev_size(clone->source_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) if (source_dev_size < clone->ti->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) dm_put_device(clone->ti, clone->source_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) *error = "Device size larger than source device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) static int copy_ctr_args(struct clone *clone, int argc, const char **argv, char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) const char **copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) if (!copy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) for (i = 0; i < argc; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) copy[i] = kstrdup(argv[i], GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) if (!copy[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) while (i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) kfree(copy[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) kfree(copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) clone->nr_ctr_args = argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) clone->ctr_args = copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) *error = "Failed to allocate memory for table line";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) static int clone_ctr(struct dm_target *ti, unsigned int argc, char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) sector_t nr_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) struct clone *clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) struct dm_arg_set as;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) if (argc < 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) ti->error = "Invalid number of arguments";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) as.argc = argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) as.argv = argv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) clone = kzalloc(sizeof(*clone), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) if (!clone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) ti->error = "Failed to allocate clone structure";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) clone->ti = ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) /* Initialize dm-clone flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) __set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) __set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) __set_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) r = parse_metadata_dev(clone, &as, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) goto out_with_clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) r = parse_dest_dev(clone, &as, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) goto out_with_meta_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) r = parse_source_dev(clone, &as, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) goto out_with_dest_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) r = parse_region_size(clone, &as, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) goto out_with_source_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) clone->region_shift = __ffs(clone->region_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) nr_regions = dm_sector_div_up(ti->len, clone->region_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) /* Check for overflow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) if (nr_regions != (unsigned long)nr_regions) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) ti->error = "Too many regions. Consider increasing the region size";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) r = -EOVERFLOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) goto out_with_source_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) clone->nr_regions = nr_regions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) r = validate_nr_regions(clone->nr_regions, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) goto out_with_source_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) r = dm_set_target_max_io_len(ti, clone->region_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) ti->error = "Failed to set max io len";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) goto out_with_source_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) r = parse_feature_args(&as, clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) goto out_with_source_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) r = parse_core_args(&as, clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) goto out_with_source_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) /* Load metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) clone->cmd = dm_clone_metadata_open(clone->metadata_dev->bdev, ti->len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) clone->region_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) if (IS_ERR(clone->cmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) ti->error = "Failed to load metadata";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) r = PTR_ERR(clone->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) goto out_with_source_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) __set_clone_mode(clone, CM_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) if (get_clone_mode(clone) != CM_WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) ti->error = "Unable to get write access to metadata, please check/repair metadata";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) r = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) goto out_with_metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) clone->last_commit_jiffies = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) /* Allocate hydration hash table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) r = hash_table_init(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) ti->error = "Failed to allocate hydration hash table";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) goto out_with_metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) atomic_set(&clone->ios_in_flight, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) init_waitqueue_head(&clone->hydration_stopped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) spin_lock_init(&clone->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) bio_list_init(&clone->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) bio_list_init(&clone->deferred_discard_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) bio_list_init(&clone->deferred_flush_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) bio_list_init(&clone->deferred_flush_completions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) clone->hydration_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) atomic_set(&clone->hydrations_in_flight, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) bio_init(&clone->flush_bio, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) if (!clone->wq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) ti->error = "Failed to allocate workqueue";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) goto out_with_ht;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) INIT_WORK(&clone->worker, do_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) INIT_DELAYED_WORK(&clone->waker, do_waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) clone->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) if (IS_ERR(clone->kcopyd_client)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) r = PTR_ERR(clone->kcopyd_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) goto out_with_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) r = mempool_init_slab_pool(&clone->hydration_pool, MIN_HYDRATIONS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) _hydration_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) ti->error = "Failed to create dm_clone_region_hydration memory pool";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) goto out_with_kcopyd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) /* Save a copy of the table line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) r = copy_ctr_args(clone, argc - 3, (const char **)argv + 3, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) goto out_with_mempool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) mutex_init(&clone->commit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) /* Enable flushes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) ti->num_flush_bios = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) ti->flush_supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) /* Enable discards */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) ti->discards_supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) ti->num_discard_bios = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) ti->private = clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) out_with_mempool:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) mempool_exit(&clone->hydration_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) out_with_kcopyd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) dm_kcopyd_client_destroy(clone->kcopyd_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) out_with_wq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) destroy_workqueue(clone->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) out_with_ht:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) hash_table_exit(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) out_with_metadata:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) dm_clone_metadata_close(clone->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) out_with_source_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) dm_put_device(ti, clone->source_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) out_with_dest_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) dm_put_device(ti, clone->dest_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) out_with_meta_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) dm_put_device(ti, clone->metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) out_with_clone:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) kfree(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) static void clone_dtr(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) struct clone *clone = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) mutex_destroy(&clone->commit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) bio_uninit(&clone->flush_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) for (i = 0; i < clone->nr_ctr_args; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) kfree(clone->ctr_args[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) kfree(clone->ctr_args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) mempool_exit(&clone->hydration_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) dm_kcopyd_client_destroy(clone->kcopyd_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) destroy_workqueue(clone->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) hash_table_exit(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) dm_clone_metadata_close(clone->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) dm_put_device(ti, clone->source_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) dm_put_device(ti, clone->dest_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) dm_put_device(ti, clone->metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) kfree(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) static void clone_postsuspend(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) struct clone *clone = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) * To successfully suspend the device:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) * - We cancel the delayed work for periodic commits and wait for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) * it to finish.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) * - We stop the background hydration, i.e. we prevent new region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) * hydrations from starting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) * - We wait for any in-flight hydrations to finish.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) * - We flush the workqueue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) * - We commit the metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) cancel_delayed_work_sync(&clone->waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) * Make sure set_bit() is ordered before atomic_read(), otherwise we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) * might race with do_hydration() and miss some started region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) * hydrations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) * This is paired with smp_mb__after_atomic() in do_hydration().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) wait_event(clone->hydration_stopped, !atomic_read(&clone->hydrations_in_flight));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) flush_workqueue(clone->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) (void) commit_metadata(clone, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) static void clone_resume(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) struct clone *clone = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) clear_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) do_waker(&clone->waker.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) static bool bdev_supports_discards(struct block_device *bdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) struct request_queue *q = bdev_get_queue(bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) return (q && blk_queue_discard(q));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) * If discard_passdown was enabled verify that the destination device supports
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) * discards. Disable discard_passdown if not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) static void disable_passdown_if_not_supported(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) struct block_device *dest_dev = clone->dest_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) struct queue_limits *dest_limits = &bdev_get_queue(dest_dev)->limits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) const char *reason = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) char buf[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) if (!bdev_supports_discards(dest_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) reason = "discard unsupported";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) else if (dest_limits->max_discard_sectors < clone->region_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) reason = "max discard sectors smaller than a region";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) if (reason) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) DMWARN("Destination device (%s) %s: Disabling discard passdown.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) bdevname(dest_dev, buf), reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) static void set_discard_limits(struct clone *clone, struct queue_limits *limits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) struct block_device *dest_bdev = clone->dest_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) struct queue_limits *dest_limits = &bdev_get_queue(dest_bdev)->limits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) /* No passdown is done so we set our own virtual limits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) limits->discard_granularity = clone->region_size << SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) limits->max_discard_sectors = round_down(UINT_MAX >> SECTOR_SHIFT, clone->region_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) * clone_iterate_devices() is stacking both the source and destination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) * device limits but discards aren't passed to the source device, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) * inherit destination's limits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) limits->max_discard_sectors = dest_limits->max_discard_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) limits->max_hw_discard_sectors = dest_limits->max_hw_discard_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) limits->discard_granularity = dest_limits->discard_granularity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) limits->discard_alignment = dest_limits->discard_alignment;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) limits->discard_misaligned = dest_limits->discard_misaligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) limits->max_discard_segments = dest_limits->max_discard_segments;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) static void clone_io_hints(struct dm_target *ti, struct queue_limits *limits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) struct clone *clone = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) u64 io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) * If the system-determined stacked limits are compatible with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) * dm-clone's region size (io_opt is a factor) do not override them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) if (io_opt_sectors < clone->region_size ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) do_div(io_opt_sectors, clone->region_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) blk_limits_io_min(limits, clone->region_size << SECTOR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) blk_limits_io_opt(limits, clone->region_size << SECTOR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) disable_passdown_if_not_supported(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) set_discard_limits(clone, limits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) static int clone_iterate_devices(struct dm_target *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) iterate_devices_callout_fn fn, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) struct clone *clone = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) struct dm_dev *dest_dev = clone->dest_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) struct dm_dev *source_dev = clone->source_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) ret = fn(ti, source_dev, 0, ti->len, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) ret = fn(ti, dest_dev, 0, ti->len, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) * dm-clone message functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) static void set_hydration_threshold(struct clone *clone, unsigned int nr_regions)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) WRITE_ONCE(clone->hydration_threshold, nr_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) * If user space sets hydration_threshold to zero then the hydration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) * will stop. If at a later time the hydration_threshold is increased
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) * we must restart the hydration process by waking up the worker.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) wake_worker(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) static void set_hydration_batch_size(struct clone *clone, unsigned int nr_regions)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) WRITE_ONCE(clone->hydration_batch_size, nr_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) static void enable_hydration(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) if (!test_and_set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) wake_worker(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) static void disable_hydration(struct clone *clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) static int clone_message(struct dm_target *ti, unsigned int argc, char **argv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) char *result, unsigned int maxlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) struct clone *clone = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) unsigned int value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) if (!argc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) if (!strcasecmp(argv[0], "enable_hydration")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) enable_hydration(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) if (!strcasecmp(argv[0], "disable_hydration")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) disable_hydration(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) if (argc != 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) if (!strcasecmp(argv[0], "hydration_threshold")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) if (kstrtouint(argv[1], 10, &value))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) set_hydration_threshold(clone, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) if (!strcasecmp(argv[0], "hydration_batch_size")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) if (kstrtouint(argv[1], 10, &value))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) set_hydration_batch_size(clone, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) DMERR("%s: Unsupported message `%s'", clone_device_name(clone), argv[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) static struct target_type clone_target = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) .name = "clone",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) .version = {1, 0, 0},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) .module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) .ctr = clone_ctr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) .dtr = clone_dtr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) .map = clone_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) .end_io = clone_endio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) .postsuspend = clone_postsuspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) .resume = clone_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) .status = clone_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) .message = clone_message,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) .io_hints = clone_io_hints,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) .iterate_devices = clone_iterate_devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) /*---------------------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) /* Module functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) static int __init dm_clone_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) _hydration_cache = KMEM_CACHE(dm_clone_region_hydration, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) if (!_hydration_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) r = dm_register_target(&clone_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) if (r < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) DMERR("Failed to register clone target");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) static void __exit dm_clone_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) dm_unregister_target(&clone_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) kmem_cache_destroy(_hydration_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) _hydration_cache = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) /* Module hooks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) module_init(dm_clone_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) module_exit(dm_clone_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) MODULE_DESCRIPTION(DM_NAME " clone target");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) MODULE_AUTHOR("Nikos Tsironis <ntsironis@arrikto.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) MODULE_LICENSE("GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags