^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright (C) 2012 Red Hat. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is released under the GPL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include "dm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "dm-bio-prison-v2.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include "dm-bio-record.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "dm-cache-metadata.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/dm-io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/dm-kcopyd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/rwsem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define DM_MSG_PREFIX "cache"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) "A percentage of time allocated for copying to and/or from cache");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * Glossary:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * oblock: index of an origin block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * cblock: index of a cache block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * promotion: movement of a block from origin to cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * demotion: movement of a block from cache to origin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * migration: movement of a block between the origin and cache device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * either direction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct io_tracker {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * Sectors of in-flight IO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) sector_t in_flight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * The time, in jiffies, when this device became idle (if it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * indeed idle).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) unsigned long idle_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) unsigned long last_update_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static void iot_init(struct io_tracker *iot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) spin_lock_init(&iot->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) iot->in_flight = 0ul;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) iot->idle_time = 0ul;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) iot->last_update_time = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static bool __iot_idle_for(struct io_tracker *iot, unsigned long jifs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (iot->in_flight)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) return time_after(jiffies, iot->idle_time + jifs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) static bool iot_idle_for(struct io_tracker *iot, unsigned long jifs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) bool r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) spin_lock_irq(&iot->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) r = __iot_idle_for(iot, jifs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) spin_unlock_irq(&iot->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static void iot_io_begin(struct io_tracker *iot, sector_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) spin_lock_irq(&iot->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) iot->in_flight += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) spin_unlock_irq(&iot->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static void __iot_io_end(struct io_tracker *iot, sector_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) if (!len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) iot->in_flight -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) if (!iot->in_flight)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) iot->idle_time = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static void iot_io_end(struct io_tracker *iot, sector_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) spin_lock_irqsave(&iot->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) __iot_io_end(iot, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) spin_unlock_irqrestore(&iot->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * Represents a chunk of future work. 'input' allows continuations to pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * values between themselves, typically error values.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct continuation {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct work_struct ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) blk_status_t input;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) static inline void init_continuation(struct continuation *k,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) void (*fn)(struct work_struct *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) INIT_WORK(&k->ws, fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) k->input = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static inline void queue_continuation(struct workqueue_struct *wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) struct continuation *k)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) queue_work(wq, &k->ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * The batcher collects together pieces of work that need a particular
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * operation to occur before they can proceed (typically a commit).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct batcher {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * The operation that everyone is waiting for.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) blk_status_t (*commit_op)(void *context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) void *commit_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * This is how bios should be issued once the commit op is complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * (accounted_request).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) void (*issue_op)(struct bio *bio, void *context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) void *issue_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * Queued work gets put on here after commit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct workqueue_struct *wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) struct list_head work_items;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct work_struct commit_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) bool commit_scheduled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static void __commit(struct work_struct *_ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct batcher *b = container_of(_ws, struct batcher, commit_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) blk_status_t r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) struct list_head work_items;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) struct work_struct *ws, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct continuation *k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) INIT_LIST_HEAD(&work_items);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * We have to grab these before the commit_op to avoid a race
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * condition.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) spin_lock_irq(&b->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) list_splice_init(&b->work_items, &work_items);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) bio_list_merge(&bios, &b->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) bio_list_init(&b->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) b->commit_scheduled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) spin_unlock_irq(&b->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) r = b->commit_op(b->commit_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) list_for_each_entry_safe(ws, tmp, &work_items, entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) k = container_of(ws, struct continuation, ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) k->input = r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) queue_work(b->wq, ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) while ((bio = bio_list_pop(&bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) bio->bi_status = r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) b->issue_op(bio, b->issue_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) static void batcher_init(struct batcher *b,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) blk_status_t (*commit_op)(void *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) void *commit_context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) void (*issue_op)(struct bio *bio, void *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) void *issue_context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) struct workqueue_struct *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) b->commit_op = commit_op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) b->commit_context = commit_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) b->issue_op = issue_op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) b->issue_context = issue_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) b->wq = wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) spin_lock_init(&b->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) INIT_LIST_HEAD(&b->work_items);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) bio_list_init(&b->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) INIT_WORK(&b->commit_work, __commit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) b->commit_scheduled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static void async_commit(struct batcher *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) queue_work(b->wq, &b->commit_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) static void continue_after_commit(struct batcher *b, struct continuation *k)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) bool commit_scheduled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) spin_lock_irq(&b->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) commit_scheduled = b->commit_scheduled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) list_add_tail(&k->ws.entry, &b->work_items);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) spin_unlock_irq(&b->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if (commit_scheduled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) async_commit(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * Bios are errored if commit failed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) static void issue_after_commit(struct batcher *b, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) bool commit_scheduled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) spin_lock_irq(&b->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) commit_scheduled = b->commit_scheduled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) bio_list_add(&b->bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) spin_unlock_irq(&b->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (commit_scheduled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) async_commit(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * Call this if some urgent work is waiting for the commit to complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) static void schedule_commit(struct batcher *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) bool immediate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) spin_lock_irq(&b->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) b->commit_scheduled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) spin_unlock_irq(&b->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if (immediate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) async_commit(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * There are a couple of places where we let a bio run, but want to do some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * work before calling its endio function. We do this by temporarily
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * changing the endio fn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) struct dm_hook_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) bio_end_io_t *bi_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) bio_end_io_t *bi_end_io, void *bi_private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) h->bi_end_io = bio->bi_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) bio->bi_end_io = bi_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) bio->bi_private = bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) bio->bi_end_io = h->bi_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) #define MIGRATION_POOL_SIZE 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) #define COMMIT_PERIOD HZ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) #define MIGRATION_COUNT_WINDOW 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * The block size of the device holding cache data must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * between 32KB and 1GB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) enum cache_metadata_mode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) CM_WRITE, /* metadata may be changed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) CM_READ_ONLY, /* metadata may not be changed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) CM_FAIL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) enum cache_io_mode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * Data is written to cached blocks only. These blocks are marked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * dirty. If you lose the cache device you will lose data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * Potential performance increase for both reads and writes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) CM_IO_WRITEBACK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * Data is written to both cache and origin. Blocks are never
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * dirty. Potential performance benfit for reads only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) CM_IO_WRITETHROUGH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * A degraded mode useful for various cache coherency situations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * (eg, rolling back snapshots). Reads and writes always go to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * origin. If a write goes to a cached oblock, then the cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * block is invalidated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) CM_IO_PASSTHROUGH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct cache_features {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) enum cache_metadata_mode mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) enum cache_io_mode io_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) unsigned metadata_version;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) bool discard_passdown:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) struct cache_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) atomic_t read_hit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) atomic_t read_miss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) atomic_t write_hit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) atomic_t write_miss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) atomic_t demotion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) atomic_t promotion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) atomic_t writeback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) atomic_t copies_avoided;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) atomic_t cache_cell_clash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) atomic_t commit_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) atomic_t discard_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) struct cache {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct dm_target *ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * Fields for converting from sectors to blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) int sectors_per_block_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) sector_t sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) struct dm_cache_metadata *cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * Metadata is written to this device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct dm_dev *metadata_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * The slower of the two data devices. Typically a spindle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) struct dm_dev *origin_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * The faster of the two data devices. Typically an SSD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) struct dm_dev *cache_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * Size of the origin device in _complete_ blocks and native sectors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) dm_oblock_t origin_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) sector_t origin_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * Size of the cache device in blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) dm_cblock_t cache_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * Invalidation fields.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) spinlock_t invalidation_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) struct list_head invalidation_requests;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) sector_t migration_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) wait_queue_head_t migration_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) atomic_t nr_allocated_migrations;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * The number of in flight migrations that are performing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * background io. eg, promotion, writeback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) atomic_t nr_io_migrations;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) struct bio_list deferred_bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) struct rw_semaphore quiesce_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * origin_blocks entries, discarded if set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) dm_dblock_t discard_nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) unsigned long *discard_bitset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) uint32_t discard_block_size; /* a power of 2 times sectors per block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * Rather than reconstructing the table line for the status we just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * save it and regurgitate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) unsigned nr_ctr_args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) const char **ctr_args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) struct dm_kcopyd_client *copier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) struct work_struct deferred_bio_worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) struct work_struct migration_worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) struct workqueue_struct *wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct delayed_work waker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) struct dm_bio_prison_v2 *prison;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * cache_size entries, dirty if set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) unsigned long *dirty_bitset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) atomic_t nr_dirty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) unsigned policy_nr_args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) struct dm_cache_policy *policy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * Cache features such as write-through.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) struct cache_features features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) struct cache_stats stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) bool need_tick_bio:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) bool sized:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) bool invalidate:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) bool commit_requested:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) bool loaded_mappings:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) bool loaded_discards:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct rw_semaphore background_work_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) struct batcher committer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) struct work_struct commit_ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) struct io_tracker tracker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) mempool_t migration_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) struct bio_set bs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) struct per_bio_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) bool tick:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) unsigned req_nr:2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) struct dm_bio_prison_cell_v2 *cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) struct dm_hook_info hook_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) sector_t len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct dm_cache_migration {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) struct continuation k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) struct cache *cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) struct policy_work *op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) struct bio *overwrite_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) struct dm_bio_prison_cell_v2 *cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) dm_cblock_t invalidate_cblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) dm_oblock_t invalidate_oblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) static bool writethrough_mode(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) return cache->features.io_mode == CM_IO_WRITETHROUGH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) static bool writeback_mode(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) return cache->features.io_mode == CM_IO_WRITEBACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) static inline bool passthrough_mode(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) static void wake_deferred_bio_worker(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) queue_work(cache->wq, &cache->deferred_bio_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) static void wake_migration_worker(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (passthrough_mode(cache))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) queue_work(cache->wq, &cache->migration_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) dm_bio_prison_free_cell_v2(cache->prison, cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) static struct dm_cache_migration *alloc_migration(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) struct dm_cache_migration *mg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) mg = mempool_alloc(&cache->migration_pool, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) memset(mg, 0, sizeof(*mg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) mg->cache = cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) atomic_inc(&cache->nr_allocated_migrations);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) return mg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) static void free_migration(struct dm_cache_migration *mg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (atomic_dec_and_test(&cache->nr_allocated_migrations))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) wake_up(&cache->migration_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) mempool_free(mg, &cache->migration_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) static inline dm_oblock_t oblock_succ(dm_oblock_t b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) return to_oblock(from_oblock(b) + 1ull);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) key->virtual = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) key->dev = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) key->block_begin = from_oblock(begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) key->block_end = from_oblock(end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) * We have two lock levels. Level 0, which is used to prevent WRITEs, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) * level 1 which prevents *both* READs and WRITEs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) #define WRITE_LOCK_LEVEL 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) #define READ_WRITE_LOCK_LEVEL 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) static unsigned lock_level(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) return bio_data_dir(bio) == WRITE ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) WRITE_LOCK_LEVEL :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) READ_WRITE_LOCK_LEVEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) * Per bio data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) static struct per_bio_data *get_per_bio_data(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) BUG_ON(!pb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) return pb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) static struct per_bio_data *init_per_bio_data(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) struct per_bio_data *pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) pb->tick = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) pb->req_nr = dm_bio_get_target_bio_nr(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) pb->cell = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) pb->len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return pb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) static void defer_bio(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) spin_lock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) bio_list_add(&cache->deferred_bios, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) spin_unlock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) wake_deferred_bio_worker(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) static void defer_bios(struct cache *cache, struct bio_list *bios)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) spin_lock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) bio_list_merge(&cache->deferred_bios, bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) bio_list_init(bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) spin_unlock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) wake_deferred_bio_worker(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) bool r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) struct per_bio_data *pb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct dm_cell_key_v2 key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) struct dm_bio_prison_cell_v2 *cell_prealloc, *cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) build_key(oblock, end, &key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) if (!r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * Failed to get the lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) free_prison_cell(cache, cell_prealloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) if (cell != cell_prealloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) free_prison_cell(cache, cell_prealloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) pb->cell = cell;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) static bool is_dirty(struct cache *cache, dm_cblock_t b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) return test_bit(from_cblock(b), cache->dirty_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) static void set_dirty(struct cache *cache, dm_cblock_t cblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) atomic_inc(&cache->nr_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) policy_set_dirty(cache->policy, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) * These two are called when setting after migrations to force the policy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) * and dirty bitset to be in sync.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) static void force_set_dirty(struct cache *cache, dm_cblock_t cblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) atomic_inc(&cache->nr_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) policy_set_dirty(cache->policy, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) if (atomic_dec_return(&cache->nr_dirty) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) dm_table_event(cache->ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) policy_clear_dirty(cache->policy, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) static bool block_size_is_power_of_two(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) return cache->sectors_per_block_shift >= 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) static dm_block_t block_div(dm_block_t b, uint32_t n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) do_div(b, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) return b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) static dm_block_t oblocks_per_dblock(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) dm_block_t oblocks = cache->discard_block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (block_size_is_power_of_two(cache))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) oblocks >>= cache->sectors_per_block_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) oblocks = block_div(oblocks, cache->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return oblocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return to_dblock(block_div(from_oblock(oblock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) oblocks_per_dblock(cache)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) static void set_discard(struct cache *cache, dm_dblock_t b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) atomic_inc(&cache->stats.discard_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) spin_lock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) set_bit(from_dblock(b), cache->discard_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) spin_unlock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) static void clear_discard(struct cache *cache, dm_dblock_t b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) spin_lock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) clear_bit(from_dblock(b), cache->discard_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) spin_unlock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) static bool is_discarded(struct cache *cache, dm_dblock_t b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) spin_lock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) r = test_bit(from_dblock(b), cache->discard_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) spin_unlock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) spin_lock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) cache->discard_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) spin_unlock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * Remapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) static void remap_to_origin(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) bio_set_dev(bio, cache->origin_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) static void remap_to_cache(struct cache *cache, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) dm_cblock_t cblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) sector_t bi_sector = bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) sector_t block = from_cblock(cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) bio_set_dev(bio, cache->cache_dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (!block_size_is_power_of_two(cache))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) bio->bi_iter.bi_sector =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) (block * cache->sectors_per_block) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) sector_div(bi_sector, cache->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) bio->bi_iter.bi_sector =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) (block << cache->sectors_per_block_shift) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) (bi_sector & (cache->sectors_per_block - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) struct per_bio_data *pb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) spin_lock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) bio_op(bio) != REQ_OP_DISCARD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) pb->tick = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) cache->need_tick_bio = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) spin_unlock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) static void __remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) dm_oblock_t oblock, bool bio_has_pbd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) if (bio_has_pbd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) check_if_tick_bio_needed(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) remap_to_origin(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) if (bio_data_dir(bio) == WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) clear_discard(cache, oblock_to_dblock(cache, oblock));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) dm_oblock_t oblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) // FIXME: check_if_tick_bio_needed() is called way too much through this interface
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) __remap_to_origin_clear_discard(cache, bio, oblock, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) static void remap_to_cache_dirty(struct cache *cache, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) dm_oblock_t oblock, dm_cblock_t cblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) check_if_tick_bio_needed(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) remap_to_cache(cache, bio, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) if (bio_data_dir(bio) == WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) set_dirty(cache, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) clear_discard(cache, oblock_to_dblock(cache, oblock));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) sector_t block_nr = bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) if (!block_size_is_power_of_two(cache))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) (void) sector_div(block_nr, cache->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) block_nr >>= cache->sectors_per_block_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) return to_oblock(block_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) static bool accountable_bio(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) return bio_op(bio) != REQ_OP_DISCARD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) static void accounted_begin(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) struct per_bio_data *pb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (accountable_bio(cache, bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) pb->len = bio_sectors(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) iot_io_begin(&cache->tracker, pb->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) static void accounted_complete(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) struct per_bio_data *pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) iot_io_end(&cache->tracker, pb->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) static void accounted_request(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) accounted_begin(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) static void issue_op(struct bio *bio, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) struct cache *cache = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) accounted_request(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * When running in writethrough mode we need to send writes to clean blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * to both the cache and origin devices. Clone the bio and send them in parallel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) dm_oblock_t oblock, dm_cblock_t cblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) struct bio *origin_bio = bio_clone_fast(bio, GFP_NOIO, &cache->bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) BUG_ON(!origin_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) bio_chain(origin_bio, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * Passing false to __remap_to_origin_clear_discard() skips
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) * all code that might use per_bio_data (since clone doesn't have it)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) __remap_to_origin_clear_discard(cache, origin_bio, oblock, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) submit_bio(origin_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) remap_to_cache(cache, bio, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * Failure modes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) static enum cache_metadata_mode get_cache_mode(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) return cache->features.mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) static const char *cache_device_name(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) return dm_table_device_name(cache->ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) const char *descs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) "write",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) "read-only",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) "fail"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) dm_table_event(cache->ti->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) DMINFO("%s: switching cache to %s mode",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) cache_device_name(cache), descs[(int)mode]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) bool needs_check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) enum cache_metadata_mode old_mode = get_cache_mode(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) DMERR("%s: unable to read needs_check flag, setting failure mode.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) new_mode = CM_FAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) if (new_mode == CM_WRITE && needs_check) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) DMERR("%s: unable to switch cache to write mode until repaired.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) if (old_mode != new_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) new_mode = old_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) new_mode = CM_READ_ONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) /* Never move out of fail mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) if (old_mode == CM_FAIL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) new_mode = CM_FAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) switch (new_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) case CM_FAIL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) case CM_READ_ONLY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) dm_cache_metadata_set_read_only(cache->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) case CM_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) dm_cache_metadata_set_read_write(cache->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) cache->features.mode = new_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) if (new_mode != old_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) notify_mode_switch(cache, new_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) static void abort_transaction(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) const char *dev_name = cache_device_name(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) if (get_cache_mode(cache) >= CM_READ_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) if (dm_cache_metadata_set_needs_check(cache->cmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) set_cache_mode(cache, CM_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) if (dm_cache_metadata_abort(cache->cmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) DMERR("%s: failed to abort metadata transaction", dev_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) set_cache_mode(cache, CM_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) static void metadata_operation_failed(struct cache *cache, const char *op, int r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) cache_device_name(cache), op, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) abort_transaction(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) set_cache_mode(cache, CM_READ_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) static void load_stats(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) struct dm_cache_statistics stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) dm_cache_metadata_get_stats(cache->cmd, &stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) atomic_set(&cache->stats.read_hit, stats.read_hits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) atomic_set(&cache->stats.read_miss, stats.read_misses);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) atomic_set(&cache->stats.write_hit, stats.write_hits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) atomic_set(&cache->stats.write_miss, stats.write_misses);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) static void save_stats(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) struct dm_cache_statistics stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (get_cache_mode(cache) >= CM_READ_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) stats.read_hits = atomic_read(&cache->stats.read_hit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) stats.read_misses = atomic_read(&cache->stats.read_miss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) stats.write_hits = atomic_read(&cache->stats.write_hit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) stats.write_misses = atomic_read(&cache->stats.write_miss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) dm_cache_metadata_set_stats(cache->cmd, &stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) static void update_stats(struct cache_stats *stats, enum policy_operation op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) switch (op) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) case POLICY_PROMOTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) atomic_inc(&stats->promotion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) case POLICY_DEMOTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) atomic_inc(&stats->demotion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) case POLICY_WRITEBACK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) atomic_inc(&stats->writeback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) * Migration processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) * Migration covers moving data from the origin device to the cache, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) * vice versa.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) static void inc_io_migrations(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) atomic_inc(&cache->nr_io_migrations);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) static void dec_io_migrations(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) atomic_dec(&cache->nr_io_migrations);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) static bool discard_or_flush(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) static void calc_discard_block_range(struct cache *cache, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) dm_dblock_t *b, dm_dblock_t *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) sector_t sb = bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) sector_t se = bio_end_sector(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) if (se - sb < cache->discard_block_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) *e = *b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) *e = to_dblock(block_div(se, cache->discard_block_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) static void prevent_background_work(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) lockdep_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) down_write(&cache->background_work_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) lockdep_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) static void allow_background_work(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) lockdep_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) up_write(&cache->background_work_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) lockdep_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) static bool background_work_begin(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) bool r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) lockdep_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) r = down_read_trylock(&cache->background_work_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) lockdep_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) static void background_work_end(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) lockdep_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) up_read(&cache->background_work_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) lockdep_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) static bool bio_writes_complete_block(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) return (bio_data_dir(bio) == WRITE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) return writeback_mode(cache) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) (is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) static void quiesce(struct dm_cache_migration *mg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) void (*continuation)(struct work_struct *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) init_continuation(&mg->k, continuation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) static struct dm_cache_migration *ws_to_mg(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) struct continuation *k = container_of(ws, struct continuation, ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) return container_of(k, struct dm_cache_migration, k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) static void copy_complete(int read_err, unsigned long write_err, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) if (read_err || write_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) mg->k.input = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) queue_continuation(mg->cache->wq, &mg->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) static void copy(struct dm_cache_migration *mg, bool promote)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) struct dm_io_region o_region, c_region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) o_region.bdev = cache->origin_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) o_region.count = cache->sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) c_region.bdev = cache->cache_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) c_region.count = cache->sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) if (promote)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) static void bio_drop_shared_lock(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) struct per_bio_data *pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) free_prison_cell(cache, pb->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) pb->cell = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) static void overwrite_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) struct dm_cache_migration *mg = bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) struct per_bio_data *pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) dm_unhook_bio(&pb->hook_info, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) if (bio->bi_status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) mg->k.input = bio->bi_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) queue_continuation(cache->wq, &mg->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) static void overwrite(struct dm_cache_migration *mg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) void (*continuation)(struct work_struct *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) struct bio *bio = mg->overwrite_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) struct per_bio_data *pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) * The overwrite bio is part of the copy operation, as such it does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) * not set/clear discard or dirty flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) if (mg->op->op == POLICY_PROMOTE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) remap_to_cache(mg->cache, bio, mg->op->cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) remap_to_origin(mg->cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) init_continuation(&mg->k, continuation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) accounted_request(mg->cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) * Migration steps:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) * 1) exclusive lock preventing WRITEs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) * 2) quiesce
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) * 3) copy or issue overwrite bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) * 4) upgrade to exclusive lock preventing READs and WRITEs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) * 5) quiesce
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) * 6) update metadata and commit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) * 7) unlock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) static void mg_complete(struct dm_cache_migration *mg, bool success)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) struct policy_work *op = mg->op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) dm_cblock_t cblock = op->cblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) if (success)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) update_stats(&cache->stats, op->op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) switch (op->op) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) case POLICY_PROMOTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) clear_discard(cache, oblock_to_dblock(cache, op->oblock));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) policy_complete_background_work(cache->policy, op, success);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) if (mg->overwrite_bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) if (success)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) force_set_dirty(cache, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) else if (mg->k.input)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) mg->overwrite_bio->bi_status = mg->k.input;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) mg->overwrite_bio->bi_status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) bio_endio(mg->overwrite_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) if (success)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) force_clear_dirty(cache, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) dec_io_migrations(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) case POLICY_DEMOTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) * We clear dirty here to update the nr_dirty counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) if (success)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) force_clear_dirty(cache, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) policy_complete_background_work(cache->policy, op, success);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) dec_io_migrations(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) case POLICY_WRITEBACK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) if (success)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) force_clear_dirty(cache, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) policy_complete_background_work(cache->policy, op, success);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) dec_io_migrations(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) if (mg->cell) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) free_prison_cell(cache, mg->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) free_migration(mg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) defer_bios(cache, &bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) wake_migration_worker(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) background_work_end(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) static void mg_success(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) struct dm_cache_migration *mg = ws_to_mg(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) mg_complete(mg, mg->k.input == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) static void mg_update_metadata(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) struct dm_cache_migration *mg = ws_to_mg(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) struct policy_work *op = mg->op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) switch (op->op) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) case POLICY_PROMOTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) DMERR_LIMIT("%s: migration failed; couldn't insert mapping",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) metadata_operation_failed(cache, "dm_cache_insert_mapping", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) mg_complete(mg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) mg_complete(mg, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) case POLICY_DEMOTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) r = dm_cache_remove_mapping(cache->cmd, op->cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) mg_complete(mg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) * It would be nice if we only had to commit when a REQ_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) * comes through. But there's one scenario that we have to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) * look out for:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) * - vblock x in a cache block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) * - domotion occurs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) * - cache block gets reallocated and over written
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) * - crash
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) * When we recover, because there was no commit the cache will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) * rollback to having the data for vblock x in the cache block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) * But the cache block has since been overwritten, so it'll end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) * up pointing to data that was never in 'x' during the history
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) * of the device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) * To avoid this issue we require a commit as part of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) * demotion operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) init_continuation(&mg->k, mg_success);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) continue_after_commit(&cache->committer, &mg->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) schedule_commit(&cache->committer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) case POLICY_WRITEBACK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) mg_complete(mg, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) static void mg_update_metadata_after_copy(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) struct dm_cache_migration *mg = ws_to_mg(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) * Did the copy succeed?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) if (mg->k.input)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) mg_complete(mg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) mg_update_metadata(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) static void mg_upgrade_lock(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) struct dm_cache_migration *mg = ws_to_mg(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) * Did the copy succeed?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) if (mg->k.input)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) mg_complete(mg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) * Now we want the lock to prevent both reads and writes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) READ_WRITE_LOCK_LEVEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) mg_complete(mg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) else if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) quiesce(mg, mg_update_metadata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) mg_update_metadata(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) static void mg_full_copy(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) struct dm_cache_migration *mg = ws_to_mg(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) struct policy_work *op = mg->op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) bool is_policy_promote = (op->op == POLICY_PROMOTE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) if ((!is_policy_promote && !is_dirty(cache, op->cblock)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) is_discarded_oblock(cache, op->oblock)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) mg_upgrade_lock(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) init_continuation(&mg->k, mg_upgrade_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) copy(mg, is_policy_promote);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) static void mg_copy(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) struct dm_cache_migration *mg = ws_to_mg(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) if (mg->overwrite_bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) * No exclusive lock was held when we last checked if the bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) * was optimisable. So we have to check again in case things
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) * have changed (eg, the block may no longer be discarded).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) * Fallback to a real full copy after doing some tidying up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) mg->overwrite_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) inc_io_migrations(mg->cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) mg_full_copy(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) * It's safe to do this here, even though it's new data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) * because all IO has been locked out of the block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) * so _not_ using mg_upgrade_lock() as continutation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) overwrite(mg, mg_update_metadata_after_copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) mg_full_copy(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) static int mg_lock_writes(struct dm_cache_migration *mg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) struct dm_cell_key_v2 key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) struct dm_bio_prison_cell_v2 *prealloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) prealloc = alloc_prison_cell(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) * Prevent writes to the block, but allow reads to continue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) * Unless we're using an overwrite bio, in which case we lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) * everything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) r = dm_cell_lock_v2(cache->prison, &key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) mg->overwrite_bio ? READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) prealloc, &mg->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) if (r < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) free_prison_cell(cache, prealloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) mg_complete(mg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) if (mg->cell != prealloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) free_prison_cell(cache, prealloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) if (r == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) mg_copy(&mg->k.ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) quiesce(mg, mg_copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) struct dm_cache_migration *mg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) if (!background_work_begin(cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) policy_complete_background_work(cache->policy, op, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) mg = alloc_migration(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) mg->op = op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) mg->overwrite_bio = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) if (!bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) inc_io_migrations(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) return mg_lock_writes(mg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) * invalidation processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) static void invalidate_complete(struct dm_cache_migration *mg, bool success)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) free_prison_cell(cache, mg->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) if (!success && mg->overwrite_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) bio_io_error(mg->overwrite_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) free_migration(mg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) defer_bios(cache, &bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) background_work_end(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) static void invalidate_completed(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) struct dm_cache_migration *mg = ws_to_mg(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) invalidate_complete(mg, !mg->k.input);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) int r = policy_invalidate_mapping(cache->policy, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) if (!r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) r = dm_cache_remove_mapping(cache->cmd, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) } else if (r == -ENODATA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) * Harmless, already unmapped.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) static void invalidate_remove(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) struct dm_cache_migration *mg = ws_to_mg(ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) r = invalidate_cblock(cache, mg->invalidate_cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) invalidate_complete(mg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) init_continuation(&mg->k, invalidate_completed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) continue_after_commit(&cache->committer, &mg->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) mg->overwrite_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) schedule_commit(&cache->committer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) static int invalidate_lock(struct dm_cache_migration *mg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) struct dm_cell_key_v2 key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) struct cache *cache = mg->cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) struct dm_bio_prison_cell_v2 *prealloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) prealloc = alloc_prison_cell(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) r = dm_cell_lock_v2(cache->prison, &key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) if (r < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) free_prison_cell(cache, prealloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) invalidate_complete(mg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) if (mg->cell != prealloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) free_prison_cell(cache, prealloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) quiesce(mg, invalidate_remove);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) * We can't call invalidate_remove() directly here because we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) * might still be in request context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) init_continuation(&mg->k, invalidate_remove);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) queue_work(cache->wq, &mg->k.ws);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) static int invalidate_start(struct cache *cache, dm_cblock_t cblock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) dm_oblock_t oblock, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) struct dm_cache_migration *mg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) if (!background_work_begin(cache))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) mg = alloc_migration(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) mg->overwrite_bio = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) mg->invalidate_cblock = cblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) mg->invalidate_oblock = oblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) return invalidate_lock(mg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) * bio processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) enum busy {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) IDLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) BUSY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) static enum busy spare_migration_bandwidth(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) bool idle = iot_idle_for(&cache->tracker, HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) cache->sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) if (idle && current_volume <= cache->migration_threshold)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) return IDLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) return BUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) static void inc_hit_counter(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) atomic_inc(bio_data_dir(bio) == READ ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) &cache->stats.read_hit : &cache->stats.write_hit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) static void inc_miss_counter(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) atomic_inc(bio_data_dir(bio) == READ ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) &cache->stats.read_miss : &cache->stats.write_miss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) bool *commit_needed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) int r, data_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) bool rb, background_queued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) dm_cblock_t cblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) *commit_needed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) rb = bio_detain_shared(cache, block, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) if (!rb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) * An exclusive lock is held for this block, so we have to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) * wait. We set the commit_needed flag so the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) * transaction will be committed asap, allowing this lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) * to be dropped.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) *commit_needed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) data_dir = bio_data_dir(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) if (optimisable_bio(cache, bio, block)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) struct policy_work *op = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) if (unlikely(r && r != -ENOENT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) cache_device_name(cache), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) if (r == -ENOENT && op) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) bio_drop_shared_lock(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) BUG_ON(op->op != POLICY_PROMOTE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) mg_start(cache, op, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) if (unlikely(r && r != -ENOENT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) DMERR_LIMIT("%s: policy_lookup() failed with r = %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) cache_device_name(cache), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) if (background_queued)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) wake_migration_worker(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) if (r == -ENOENT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) struct per_bio_data *pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) * Miss.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) inc_miss_counter(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) if (pb->req_nr == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) accounted_begin(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) remap_to_origin_clear_discard(cache, bio, block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) * This is a duplicate writethrough io that is no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) * longer needed because the block has been demoted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) * Hit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) inc_hit_counter(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) * Passthrough always maps to the origin, invalidating any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) * cache blocks that are written to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) if (passthrough_mode(cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) if (bio_data_dir(bio) == WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) bio_drop_shared_lock(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) atomic_inc(&cache->stats.demotion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) invalidate_start(cache, cblock, block, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) remap_to_origin_clear_discard(cache, bio, block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) !is_dirty(cache, cblock)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) remap_to_origin_and_cache(cache, bio, block, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) accounted_begin(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) remap_to_cache_dirty(cache, bio, block, cblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) * dm core turns FUA requests into a separate payload and FLUSH req.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) if (bio->bi_opf & REQ_FUA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) * issue_after_commit will call accounted_begin a second time. So
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) * we call accounted_complete() to avoid double accounting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) accounted_complete(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) issue_after_commit(&cache->committer, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) *commit_needed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) return DM_MAPIO_REMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) static bool process_bio(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) bool commit_needed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) return commit_needed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) * A non-zero return indicates read_only or fail_io mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) static int commit(struct cache *cache, bool clean_shutdown)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) if (get_cache_mode(cache) >= CM_READ_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) atomic_inc(&cache->stats.commit_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) r = dm_cache_commit(cache->cmd, clean_shutdown);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) metadata_operation_failed(cache, "dm_cache_commit", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) * Used by the batcher.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) static blk_status_t commit_op(void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) struct cache *cache = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) if (dm_cache_changed_this_transaction(cache->cmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) return errno_to_blk_status(commit(cache, false));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) static bool process_flush_bio(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) struct per_bio_data *pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) if (!pb->req_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) remap_to_origin(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) remap_to_cache(cache, bio, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) issue_after_commit(&cache->committer, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) static bool process_discard_bio(struct cache *cache, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) dm_dblock_t b, e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) // FIXME: do we need to lock the region? Or can we just assume the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) // user wont be so foolish as to issue discard concurrently with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) // other IO?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) calc_discard_block_range(cache, bio, &b, &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) while (b != e) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) set_discard(cache, b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) b = to_dblock(from_dblock(b) + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) if (cache->features.discard_passdown) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) remap_to_origin(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) static void process_deferred_bios(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) struct cache *cache = container_of(ws, struct cache, deferred_bio_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) bool commit_needed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) spin_lock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) bio_list_merge(&bios, &cache->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) bio_list_init(&cache->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) spin_unlock_irq(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) while ((bio = bio_list_pop(&bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) if (bio->bi_opf & REQ_PREFLUSH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) commit_needed = process_flush_bio(cache, bio) || commit_needed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) else if (bio_op(bio) == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) commit_needed = process_discard_bio(cache, bio) || commit_needed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) commit_needed = process_bio(cache, bio) || commit_needed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) if (commit_needed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) schedule_commit(&cache->committer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) * Main worker loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) static void requeue_deferred_bios(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) struct bio_list bios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) bio_list_init(&bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) bio_list_merge(&bios, &cache->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) bio_list_init(&cache->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) while ((bio = bio_list_pop(&bios))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) bio->bi_status = BLK_STS_DM_REQUEUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) * We want to commit periodically so that not too much
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) * unwritten metadata builds up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) static void do_waker(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) policy_tick(cache->policy, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) wake_migration_worker(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) schedule_commit(&cache->committer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) static void check_migrations(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) struct policy_work *op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) struct cache *cache = container_of(ws, struct cache, migration_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) enum busy b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) b = spare_migration_bandwidth(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) r = policy_get_background_work(cache->policy, b == IDLE, &op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) if (r == -ENODATA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) DMERR_LIMIT("%s: policy_background_work failed",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) r = mg_start(cache, op, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) /*----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) * Target methods
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) *--------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) * This function gets called on the error paths of the constructor, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) * have to cope with a partially initialised struct.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) static void destroy(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) mempool_exit(&cache->migration_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) if (cache->prison)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) dm_bio_prison_destroy_v2(cache->prison);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) if (cache->wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) destroy_workqueue(cache->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) if (cache->dirty_bitset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) free_bitset(cache->dirty_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) if (cache->discard_bitset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) free_bitset(cache->discard_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) if (cache->copier)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) dm_kcopyd_client_destroy(cache->copier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) if (cache->cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) dm_cache_metadata_close(cache->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) if (cache->metadata_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) dm_put_device(cache->ti, cache->metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) if (cache->origin_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) dm_put_device(cache->ti, cache->origin_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) if (cache->cache_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) dm_put_device(cache->ti, cache->cache_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) if (cache->policy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) dm_cache_policy_destroy(cache->policy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) for (i = 0; i < cache->nr_ctr_args ; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) kfree(cache->ctr_args[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) kfree(cache->ctr_args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) bioset_exit(&cache->bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) kfree(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) static void cache_dtr(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) destroy(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) static sector_t get_dev_size(struct dm_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) * Construct a cache device mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) * cache <metadata dev> <cache dev> <origin dev> <block size>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) * <#feature args> [<feature arg>]*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) * <policy> <#policy args> [<policy arg>]*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) * metadata dev : fast device holding the persistent metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) * cache dev : fast device holding cached data blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) * origin dev : slow device holding original data blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) * block size : cache unit size in sectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) * #feature args : number of feature arguments passed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) * feature args : writethrough. (The default is writeback.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) * policy : the replacement policy to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) * #policy args : an even number of policy arguments corresponding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) * to key/value pairs passed to the policy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) * policy args : key/value pairs passed to the policy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) * E.g. 'sequential_threshold 1024'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) * See cache-policies.txt for details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) * Optional feature arguments are:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) * writethrough : write through caching that prohibits cache block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) * content from being different from origin block content.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) * Without this argument, the default behaviour is to write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) * back cache block contents later for performance reasons,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) * so they may differ from the corresponding origin blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) struct cache_args {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) struct dm_target *ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) struct dm_dev *metadata_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) struct dm_dev *cache_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) sector_t cache_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) struct dm_dev *origin_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) sector_t origin_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) uint32_t block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) const char *policy_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) int policy_argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) const char **policy_argv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) struct cache_features features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) static void destroy_cache_args(struct cache_args *ca)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) if (ca->metadata_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) dm_put_device(ca->ti, ca->metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) if (ca->cache_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) dm_put_device(ca->ti, ca->cache_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) if (ca->origin_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) dm_put_device(ca->ti, ca->origin_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) kfree(ca);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) static bool at_least_one_arg(struct dm_arg_set *as, char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) if (!as->argc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) *error = "Insufficient args";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) sector_t metadata_dev_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) if (!at_least_one_arg(as, error))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) &ca->metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) *error = "Error opening metadata device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) metadata_dev_size = get_dev_size(ca->metadata_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) bdevname(ca->metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) if (!at_least_one_arg(as, error))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) &ca->cache_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) *error = "Error opening cache device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) ca->cache_sectors = get_dev_size(ca->cache_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) if (!at_least_one_arg(as, error))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) &ca->origin_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) *error = "Error opening origin device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) ca->origin_sectors = get_dev_size(ca->origin_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) if (ca->ti->len > ca->origin_sectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) *error = "Device size larger than cached device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) unsigned long block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) if (!at_least_one_arg(as, error))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) *error = "Invalid data block size";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) if (block_size > ca->cache_sectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) *error = "Data block size is larger than the cache device";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) ca->block_size = block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) static void init_features(struct cache_features *cf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) cf->mode = CM_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) cf->io_mode = CM_IO_WRITEBACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) cf->metadata_version = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) cf->discard_passdown = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) static const struct dm_arg _args[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) {0, 3, "Invalid number of cache feature arguments"},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) int r, mode_ctr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) unsigned argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) const char *arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) struct cache_features *cf = &ca->features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) init_features(cf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) r = dm_read_arg_group(_args, as, &argc, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) while (argc--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) arg = dm_shift_arg(as);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) if (!strcasecmp(arg, "writeback")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) cf->io_mode = CM_IO_WRITEBACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) mode_ctr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) else if (!strcasecmp(arg, "writethrough")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) cf->io_mode = CM_IO_WRITETHROUGH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) mode_ctr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) else if (!strcasecmp(arg, "passthrough")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) cf->io_mode = CM_IO_PASSTHROUGH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) mode_ctr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) else if (!strcasecmp(arg, "metadata2"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) cf->metadata_version = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) else if (!strcasecmp(arg, "no_discard_passdown"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) cf->discard_passdown = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) *error = "Unrecognised cache feature requested";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) if (mode_ctr > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) *error = "Duplicate cache io_mode features requested";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) static const struct dm_arg _args[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) {0, 1024, "Invalid number of policy arguments"},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) if (!at_least_one_arg(as, error))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) ca->policy_name = dm_shift_arg(as);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) r = dm_read_arg_group(_args, as, &ca->policy_argc, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) ca->policy_argv = (const char **)as->argv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) dm_consume_args(as, ca->policy_argc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) static int parse_cache_args(struct cache_args *ca, int argc, char **argv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) struct dm_arg_set as;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) as.argc = argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) as.argv = argv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) r = parse_metadata_dev(ca, &as, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) r = parse_cache_dev(ca, &as, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) r = parse_origin_dev(ca, &as, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) r = parse_block_size(ca, &as, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) r = parse_features(ca, &as, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) r = parse_policy(ca, &as, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) static struct kmem_cache *migration_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) #define NOT_CORE_OPTION 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) static int process_config_option(struct cache *cache, const char *key, const char *value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) unsigned long tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) if (!strcasecmp(key, "migration_threshold")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) if (kstrtoul(value, 10, &tmp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) cache->migration_threshold = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) return NOT_CORE_OPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) static int set_config_value(struct cache *cache, const char *key, const char *value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) int r = process_config_option(cache, key, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) if (r == NOT_CORE_OPTION)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) r = policy_set_config_value(cache->policy, key, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) DMWARN("bad config value for %s: %s", key, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) static int set_config_values(struct cache *cache, int argc, const char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) if (argc & 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) while (argc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) r = set_config_value(cache, argv[0], argv[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) argc -= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) argv += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) static int create_cache_policy(struct cache *cache, struct cache_args *ca,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) char **error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) cache->cache_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) cache->origin_sectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) cache->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) if (IS_ERR(p)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) *error = "Error creating cache's policy";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) return PTR_ERR(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) cache->policy = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) BUG_ON(!cache->policy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) * We want the discard block size to be at least the size of the cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) * block size and have no more than 2^14 discard blocks across the origin.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) #define MAX_DISCARD_BLOCKS (1 << 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) static bool too_many_discard_blocks(sector_t discard_block_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) sector_t origin_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) (void) sector_div(origin_size, discard_block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) return origin_size > MAX_DISCARD_BLOCKS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) static sector_t calculate_discard_block_size(sector_t cache_block_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) sector_t origin_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) sector_t discard_block_size = cache_block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) if (origin_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) while (too_many_discard_blocks(discard_block_size, origin_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) discard_block_size *= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) return discard_block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) static void set_cache_size(struct cache *cache, dm_cblock_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) dm_block_t nr_blocks = from_cblock(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) if (nr_blocks > (1 << 20) && cache->cache_size != size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) "Please consider increasing the cache block size to reduce the overall cache block count.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) (unsigned long long) nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) cache->cache_size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) #define DEFAULT_MIGRATION_THRESHOLD 2048
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) static int cache_create(struct cache_args *ca, struct cache **result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) char **error = &ca->ti->error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) struct cache *cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) struct dm_target *ti = ca->ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) dm_block_t origin_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) struct dm_cache_metadata *cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) bool may_format = ca->features.mode == CM_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) cache = kzalloc(sizeof(*cache), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) if (!cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) cache->ti = ca->ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) ti->private = cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) ti->num_flush_bios = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) ti->flush_supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) ti->num_discard_bios = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) ti->discards_supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) ti->per_io_data_size = sizeof(struct per_bio_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) cache->features = ca->features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) if (writethrough_mode(cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) /* Create bioset for writethrough bios issued to origin */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) r = bioset_init(&cache->bs, BIO_POOL_SIZE, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) cache->metadata_dev = ca->metadata_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) cache->origin_dev = ca->origin_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) cache->cache_dev = ca->cache_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) origin_blocks = cache->origin_sectors = ca->origin_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) origin_blocks = block_div(origin_blocks, ca->block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) cache->origin_blocks = to_oblock(origin_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) cache->sectors_per_block = ca->block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) if (ca->block_size & (ca->block_size - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) dm_block_t cache_size = ca->cache_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) cache->sectors_per_block_shift = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) cache_size = block_div(cache_size, ca->block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) set_cache_size(cache, to_cblock(cache_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) cache->sectors_per_block_shift = __ffs(ca->block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) r = create_cache_policy(cache, ca, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) cache->policy_nr_args = ca->policy_argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) r = set_config_values(cache, ca->policy_argc, ca->policy_argv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) *error = "Error setting cache policy's config values";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) cmd = dm_cache_metadata_open(cache->metadata_dev->bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) ca->block_size, may_format,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) dm_cache_policy_get_hint_size(cache->policy),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) ca->features.metadata_version);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) if (IS_ERR(cmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) *error = "Error creating metadata object";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) r = PTR_ERR(cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) cache->cmd = cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) set_cache_mode(cache, CM_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) if (get_cache_mode(cache) != CM_WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) *error = "Unable to get write access to metadata, please check/repair metadata.";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) if (passthrough_mode(cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) bool all_clean;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) r = dm_cache_metadata_all_clean(cache->cmd, &all_clean);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) *error = "dm_cache_metadata_all_clean() failed";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) if (!all_clean) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) *error = "Cannot enter passthrough mode unless all blocks are clean";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) policy_allow_migrations(cache->policy, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) spin_lock_init(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) bio_list_init(&cache->deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) atomic_set(&cache->nr_allocated_migrations, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) atomic_set(&cache->nr_io_migrations, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) init_waitqueue_head(&cache->migration_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) atomic_set(&cache->nr_dirty, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) if (!cache->dirty_bitset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) *error = "could not allocate dirty bitset";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) cache->discard_block_size =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) calculate_discard_block_size(cache->sectors_per_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) cache->origin_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) cache->discard_block_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) if (!cache->discard_bitset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) *error = "could not allocate discard bitset";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) if (IS_ERR(cache->copier)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) *error = "could not create kcopyd client";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) r = PTR_ERR(cache->copier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) if (!cache->wq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) *error = "could not create workqueue for metadata object";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) INIT_WORK(&cache->migration_worker, check_migrations);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) INIT_DELAYED_WORK(&cache->waker, do_waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) cache->prison = dm_bio_prison_create_v2(cache->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) if (!cache->prison) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) *error = "could not create bio prison";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) r = mempool_init_slab_pool(&cache->migration_pool, MIGRATION_POOL_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) migration_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) *error = "Error creating cache's migration mempool";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) cache->need_tick_bio = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) cache->sized = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) cache->invalidate = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) cache->commit_requested = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) cache->loaded_mappings = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) cache->loaded_discards = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) load_stats(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) atomic_set(&cache->stats.demotion, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) atomic_set(&cache->stats.promotion, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) atomic_set(&cache->stats.copies_avoided, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) atomic_set(&cache->stats.cache_cell_clash, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) atomic_set(&cache->stats.commit_count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) atomic_set(&cache->stats.discard_count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) spin_lock_init(&cache->invalidation_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) INIT_LIST_HEAD(&cache->invalidation_requests);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) batcher_init(&cache->committer, commit_op, cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) issue_op, cache, cache->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) iot_init(&cache->tracker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) init_rwsem(&cache->background_work_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) prevent_background_work(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) *result = cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) bad:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) destroy(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) static int copy_ctr_args(struct cache *cache, int argc, const char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) const char **copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) if (!copy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) for (i = 0; i < argc; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) copy[i] = kstrdup(argv[i], GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) if (!copy[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) while (i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) kfree(copy[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) kfree(copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) cache->nr_ctr_args = argc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) cache->ctr_args = copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) static int cache_ctr(struct dm_target *ti, unsigned argc, char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) int r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) struct cache_args *ca;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) struct cache *cache = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) ca = kzalloc(sizeof(*ca), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) if (!ca) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) ti->error = "Error allocating memory for cache";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) ca->ti = ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) r = parse_cache_args(ca, argc, argv, &ti->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) r = cache_create(ca, &cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) destroy(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) ti->private = cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) destroy_cache_args(ca);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) static int cache_map(struct dm_target *ti, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) bool commit_needed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) dm_oblock_t block = get_bio_block(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) init_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) * This can only occur if the io goes to a partial block at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) * the end of the origin device. We don't cache these.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) * Just remap to the origin and carry on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) remap_to_origin(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) accounted_begin(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) return DM_MAPIO_REMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) if (discard_or_flush(bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) defer_bio(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) return DM_MAPIO_SUBMITTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) r = map_bio(cache, bio, block, &commit_needed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) if (commit_needed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) schedule_commit(&cache->committer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) static int cache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) struct per_bio_data *pb = get_per_bio_data(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) if (pb->tick) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) policy_tick(cache->policy, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) spin_lock_irqsave(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) cache->need_tick_bio = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) spin_unlock_irqrestore(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) bio_drop_shared_lock(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) accounted_complete(cache, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) return DM_ENDIO_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) static int write_dirty_bitset(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) if (get_cache_mode(cache) >= CM_READ_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) static int write_discard_bitset(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) unsigned i, r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) if (get_cache_mode(cache) >= CM_READ_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) cache->discard_nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) r = dm_cache_set_discard(cache->cmd, to_dblock(i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) is_discarded(cache, to_dblock(i)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) metadata_operation_failed(cache, "dm_cache_set_discard", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) static int write_hints(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) if (get_cache_mode(cache) >= CM_READ_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) r = dm_cache_write_hints(cache->cmd, cache->policy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) metadata_operation_failed(cache, "dm_cache_write_hints", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) * returns true on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) static bool sync_metadata(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) int r1, r2, r3, r4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) r1 = write_dirty_bitset(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) if (r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) DMERR("%s: could not write dirty bitset", cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) r2 = write_discard_bitset(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) if (r2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) DMERR("%s: could not write discard bitset", cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) save_stats(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) r3 = write_hints(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) if (r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) DMERR("%s: could not write hints", cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) * If writing the above metadata failed, we still commit, but don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) * set the clean shutdown flag. This will effectively force every
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) * dirty bit to be set on reload.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) r4 = commit(cache, !r1 && !r2 && !r3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) if (r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) DMERR("%s: could not write cache metadata", cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) return !r1 && !r2 && !r3 && !r4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) static void cache_postsuspend(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) prevent_background_work(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) BUG_ON(atomic_read(&cache->nr_io_migrations));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) cancel_delayed_work_sync(&cache->waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) drain_workqueue(cache->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) WARN_ON(cache->tracker.in_flight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) * If it's a flush suspend there won't be any deferred bios, so this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) * call is harmless.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) requeue_deferred_bios(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) if (get_cache_mode(cache) == CM_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) (void) sync_metadata(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) bool dirty, uint32_t hint, bool hint_valid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) struct cache *cache = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) if (dirty) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) set_bit(from_cblock(cblock), cache->dirty_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) atomic_inc(&cache->nr_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) clear_bit(from_cblock(cblock), cache->dirty_bitset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) r = policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) * The discard block size in the on disk metadata is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) * neccessarily the same as we're currently using. So we have to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) * be careful to only set the discarded attribute if we know it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) * covers a complete block of the new size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) struct discard_load_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) struct cache *cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) * These blocks are sized using the on disk dblock size, rather
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) * than the current one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) dm_block_t block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) dm_block_t discard_begin, discard_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) static void discard_load_info_init(struct cache *cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) struct discard_load_info *li)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) li->cache = cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) li->discard_begin = li->discard_end = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) static void set_discard_range(struct discard_load_info *li)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) sector_t b, e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) if (li->discard_begin == li->discard_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) * Convert to sectors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) b = li->discard_begin * li->block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) e = li->discard_end * li->block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) * Then convert back to the current dblock size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) b = dm_sector_div_up(b, li->cache->discard_block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) sector_div(e, li->cache->discard_block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) * The origin may have shrunk, so we need to check we're still in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) * bounds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) if (e > from_dblock(li->cache->discard_nr_blocks))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) e = from_dblock(li->cache->discard_nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) for (; b < e; b++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) set_discard(li->cache, to_dblock(b));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) static int load_discard(void *context, sector_t discard_block_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) dm_dblock_t dblock, bool discard)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) struct discard_load_info *li = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) li->block_size = discard_block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) if (discard) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) if (from_dblock(dblock) == li->discard_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) * We're already in a discard range, just extend it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) li->discard_end = li->discard_end + 1ULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) * Emit the old range and start a new one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) set_discard_range(li);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) li->discard_begin = from_dblock(dblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) li->discard_end = li->discard_begin + 1ULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) set_discard_range(li);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) li->discard_begin = li->discard_end = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) static dm_cblock_t get_cache_dev_size(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) sector_t size = get_dev_size(cache->cache_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) (void) sector_div(size, cache->sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) return to_cblock(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) static bool can_resize(struct cache *cache, dm_cblock_t new_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) if (from_cblock(new_size) > from_cblock(cache->cache_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) if (cache->sized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) DMERR("%s: unable to extend cache due to missing cache table reload",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) * We can't drop a dirty block when shrinking the cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) while (from_cblock(new_size) < from_cblock(cache->cache_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) new_size = to_cblock(from_cblock(new_size) + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) if (is_dirty(cache, new_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) DMERR("%s: unable to shrink cache; cache block %llu is dirty",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) cache_device_name(cache),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) (unsigned long long) from_cblock(new_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) r = dm_cache_resize(cache->cmd, new_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) DMERR("%s: could not resize cache metadata", cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) metadata_operation_failed(cache, "dm_cache_resize", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) set_cache_size(cache, new_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) static int cache_preresume(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) dm_cblock_t csize = get_cache_dev_size(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) * Check to see if the cache has resized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) if (!cache->sized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) r = resize_cache_dev(cache, csize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) cache->sized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) } else if (csize != cache->cache_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) if (!can_resize(cache, csize))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) r = resize_cache_dev(cache, csize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) if (!cache->loaded_mappings) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) r = dm_cache_load_mappings(cache->cmd, cache->policy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) load_mapping, cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) DMERR("%s: could not load cache mappings", cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) metadata_operation_failed(cache, "dm_cache_load_mappings", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) cache->loaded_mappings = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) if (!cache->loaded_discards) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) struct discard_load_info li;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) * The discard bitset could have been resized, or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) * discard block size changed. To be safe we start by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) * setting every dblock to not discarded.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) discard_load_info_init(cache, &li);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) r = dm_cache_load_discards(cache->cmd, load_discard, &li);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) DMERR("%s: could not load origin discards", cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) metadata_operation_failed(cache, "dm_cache_load_discards", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) set_discard_range(&li);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) cache->loaded_discards = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) static void cache_resume(struct dm_target *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) cache->need_tick_bio = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) allow_background_work(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) do_waker(&cache->waker.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) static void emit_flags(struct cache *cache, char *result,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) unsigned maxlen, ssize_t *sz_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) ssize_t sz = *sz_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) struct cache_features *cf = &cache->features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) unsigned count = (cf->metadata_version == 2) + !cf->discard_passdown + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) DMEMIT("%u ", count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) if (cf->metadata_version == 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) DMEMIT("metadata2 ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) if (writethrough_mode(cache))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) DMEMIT("writethrough ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) else if (passthrough_mode(cache))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) DMEMIT("passthrough ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) else if (writeback_mode(cache))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) DMEMIT("writeback ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) DMEMIT("unknown ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) DMERR("%s: internal error: unknown io mode: %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) cache_device_name(cache), (int) cf->io_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) if (!cf->discard_passdown)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) DMEMIT("no_discard_passdown ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) *sz_ptr = sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) * Status format:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) * <cache block size> <#used cache blocks>/<#total cache blocks>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) * <#read hits> <#read misses> <#write hits> <#write misses>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) * <#demotions> <#promotions> <#dirty>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) * <#features> <features>*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) * <#core args> <core args>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) static void cache_status(struct dm_target *ti, status_type_t type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) unsigned status_flags, char *result, unsigned maxlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) ssize_t sz = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) dm_block_t nr_free_blocks_metadata = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) dm_block_t nr_blocks_metadata = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) char buf[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) dm_cblock_t residency;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) bool needs_check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) case STATUSTYPE_INFO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) if (get_cache_mode(cache) == CM_FAIL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) DMEMIT("Fail");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) /* Commit to ensure statistics aren't out-of-date */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) (void) commit(cache, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) DMERR("%s: dm_cache_get_free_metadata_block_count returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) cache_device_name(cache), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) DMERR("%s: dm_cache_get_metadata_dev_size returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) cache_device_name(cache), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) residency = policy_residency(cache->policy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) (unsigned)DM_CACHE_METADATA_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) (unsigned long long)nr_blocks_metadata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) (unsigned long long)cache->sectors_per_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) (unsigned long long) from_cblock(residency),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) (unsigned long long) from_cblock(cache->cache_size),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) (unsigned) atomic_read(&cache->stats.read_hit),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) (unsigned) atomic_read(&cache->stats.read_miss),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) (unsigned) atomic_read(&cache->stats.write_hit),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) (unsigned) atomic_read(&cache->stats.write_miss),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) (unsigned) atomic_read(&cache->stats.demotion),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) (unsigned) atomic_read(&cache->stats.promotion),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) (unsigned long) atomic_read(&cache->nr_dirty));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) emit_flags(cache, result, maxlen, &sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) DMEMIT("%s ", dm_cache_policy_get_name(cache->policy));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) if (sz < maxlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) r = policy_emit_config_values(cache->policy, result, maxlen, &sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) DMERR("%s: policy_emit_config_values returned %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) cache_device_name(cache), r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) if (get_cache_mode(cache) == CM_READ_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) DMEMIT("ro ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) DMEMIT("rw ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) r = dm_cache_metadata_needs_check(cache->cmd, &needs_check);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) if (r || needs_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) DMEMIT("needs_check ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) DMEMIT("- ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) case STATUSTYPE_TABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) DMEMIT("%s ", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) DMEMIT("%s ", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) DMEMIT("%s", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) for (i = 0; i < cache->nr_ctr_args - 1; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) DMEMIT(" %s", cache->ctr_args[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) if (cache->nr_ctr_args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) DMEMIT("Error");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) * Defines a range of cblocks, begin to (end - 1) are in the range. end is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) * the one-past-the-end value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) struct cblock_range {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) dm_cblock_t begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) dm_cblock_t end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) * A cache block range can take two forms:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) * i) A single cblock, eg. '3456'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) * ii) A begin and end cblock with a dash between, eg. 123-234
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) static int parse_cblock_range(struct cache *cache, const char *str,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) struct cblock_range *result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) char dummy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) uint64_t b, e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) * Try and parse form (ii) first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) if (r == 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) result->begin = to_cblock(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) result->end = to_cblock(e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) * That didn't work, try form (i).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) r = sscanf(str, "%llu%c", &b, &dummy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) if (r == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) result->begin = to_cblock(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) result->end = to_cblock(from_cblock(result->begin) + 1u);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) static int validate_cblock_range(struct cache *cache, struct cblock_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) uint64_t b = from_cblock(range->begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) uint64_t e = from_cblock(range->end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) uint64_t n = from_cblock(cache->cache_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) if (b >= n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) DMERR("%s: begin cblock out of range: %llu >= %llu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) cache_device_name(cache), b, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) if (e > n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) DMERR("%s: end cblock out of range: %llu > %llu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) cache_device_name(cache), e, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) if (b >= e) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) DMERR("%s: invalid cblock range: %llu >= %llu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) cache_device_name(cache), b, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) static inline dm_cblock_t cblock_succ(dm_cblock_t b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) return to_cblock(from_cblock(b) + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) static int request_invalidation(struct cache *cache, struct cblock_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) * We don't need to do any locking here because we know we're in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) * passthrough mode. There's is potential for a race between an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) * invalidation triggered by an io and an invalidation message. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) * is harmless, we must not worry if the policy call fails.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) while (range->begin != range->end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) r = invalidate_cblock(cache, range->begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) range->begin = cblock_succ(range->begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) cache->commit_requested = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) static int process_invalidate_cblocks_message(struct cache *cache, unsigned count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) const char **cblock_ranges)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) struct cblock_range range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) if (!passthrough_mode(cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) DMERR("%s: cache has to be in passthrough mode for invalidation",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) for (i = 0; i < count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) r = parse_cblock_range(cache, cblock_ranges[i], &range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) r = validate_cblock_range(cache, &range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) * Pass begin and end origin blocks to the worker and wake it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) r = request_invalidation(cache, &range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) * Supports
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) * "<key> <value>"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) * and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) * "invalidate_cblocks [(<begin>)|(<begin>-<end>)]*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) * The key migration_threshold is supported by the cache target core.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) static int cache_message(struct dm_target *ti, unsigned argc, char **argv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) char *result, unsigned maxlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) if (!argc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) if (get_cache_mode(cache) >= CM_READ_ONLY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) cache_device_name(cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) if (!strcasecmp(argv[0], "invalidate_cblocks"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) if (argc != 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) return set_config_value(cache, argv[0], argv[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) static int cache_iterate_devices(struct dm_target *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) iterate_devices_callout_fn fn, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) if (!r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) r = fn(ti, cache->origin_dev, 0, ti->len, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) static bool origin_dev_supports_discard(struct block_device *origin_bdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) struct request_queue *q = bdev_get_queue(origin_bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) return q && blk_queue_discard(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) * If discard_passdown was enabled verify that the origin device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) * supports discards. Disable discard_passdown if not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) static void disable_passdown_if_not_supported(struct cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) struct block_device *origin_bdev = cache->origin_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) const char *reason = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) char buf[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) if (!cache->features.discard_passdown)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) if (!origin_dev_supports_discard(origin_bdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) reason = "discard unsupported";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) else if (origin_limits->max_discard_sectors < cache->sectors_per_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) reason = "max discard sectors smaller than a block";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) if (reason) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) DMWARN("Origin device (%s) %s: Disabling discard passdown.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) bdevname(origin_bdev, buf), reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) cache->features.discard_passdown = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) struct block_device *origin_bdev = cache->origin_dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) if (!cache->features.discard_passdown) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) /* No passdown is done so setting own virtual limits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) cache->origin_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) * cache_iterate_devices() is stacking both origin and fast device limits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) * but discards aren't passed to fast device, so inherit origin's limits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) limits->max_discard_sectors = origin_limits->max_discard_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) limits->max_hw_discard_sectors = origin_limits->max_hw_discard_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) limits->discard_granularity = origin_limits->discard_granularity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) limits->discard_alignment = origin_limits->discard_alignment;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) limits->discard_misaligned = origin_limits->discard_misaligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) struct cache *cache = ti->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) * If the system-determined stacked limits are compatible with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) * cache's blocksize (io_opt is a factor) do not override them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) if (io_opt_sectors < cache->sectors_per_block ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) do_div(io_opt_sectors, cache->sectors_per_block)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) disable_passdown_if_not_supported(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465) set_discard_limits(cache, limits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) /*----------------------------------------------------------------*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) static struct target_type cache_target = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) .name = "cache",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) .version = {2, 2, 0},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) .module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) .ctr = cache_ctr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) .dtr = cache_dtr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) .map = cache_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) .end_io = cache_end_io,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) .postsuspend = cache_postsuspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) .preresume = cache_preresume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) .resume = cache_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) .status = cache_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) .message = cache_message,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) .iterate_devices = cache_iterate_devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) .io_hints = cache_io_hints,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) static int __init dm_cache_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) migration_cache = KMEM_CACHE(dm_cache_migration, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) if (!migration_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) r = dm_register_target(&cache_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) DMERR("cache target registration failed: %d", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) kmem_cache_destroy(migration_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) static void __exit dm_cache_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) dm_unregister_target(&cache_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) kmem_cache_destroy(migration_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) module_init(dm_cache_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) module_exit(dm_cache_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) MODULE_DESCRIPTION(DM_NAME " cache target");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) MODULE_LICENSE("GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags