^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2017 Western Digital Corporation or its affiliates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This file is released under the GPL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "dm-zoned.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #define DM_MSG_PREFIX "zoned reclaim"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) struct dmz_reclaim {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) struct dmz_metadata *metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) struct delayed_work work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) struct workqueue_struct *wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) struct dm_kcopyd_client *kc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) struct dm_kcopyd_throttle kc_throttle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) int kc_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) int dev_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /* Last target access time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) unsigned long atime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * Reclaim state flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) DMZ_RECLAIM_KCOPY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * Number of seconds of target BIO inactivity to consider the target idle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define DMZ_IDLE_PERIOD (10UL * HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * Percentage of unmapped (free) random zones below which reclaim starts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * even if the target is busy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define DMZ_RECLAIM_LOW_UNMAP_ZONES 30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * Percentage of unmapped (free) random zones above which reclaim will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * stop if the target is busy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define DMZ_RECLAIM_HIGH_UNMAP_ZONES 50
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * Align a sequential zone write pointer to chunk_block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static int dmz_reclaim_align_wp(struct dmz_reclaim *zrc, struct dm_zone *zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) sector_t block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct dmz_metadata *zmd = zrc->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct dmz_dev *dev = zone->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) sector_t wp_block = zone->wp_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) unsigned int nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (wp_block == block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) if (wp_block > block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * Zeroout the space between the write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * pointer and the requested position.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) nr_blocks = block - wp_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) ret = blkdev_issue_zeroout(dev->bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) dmz_start_sect(zmd, zone) + dmz_blk2sect(wp_block),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) dmz_blk2sect(nr_blocks), GFP_NOIO, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) dmz_dev_err(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) "Align zone %u wp %llu to %llu (wp+%u) blocks failed %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) zone->id, (unsigned long long)wp_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) (unsigned long long)block, nr_blocks, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) dmz_check_bdev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) zone->wp_block = block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * dm_kcopyd_copy end notification.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static void dmz_reclaim_kcopy_end(int read_err, unsigned long write_err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct dmz_reclaim *zrc = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (read_err || write_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) zrc->kc_err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) zrc->kc_err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) clear_bit_unlock(DMZ_RECLAIM_KCOPY, &zrc->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) wake_up_bit(&zrc->flags, DMZ_RECLAIM_KCOPY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * Copy valid blocks of src_zone into dst_zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static int dmz_reclaim_copy(struct dmz_reclaim *zrc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct dm_zone *src_zone, struct dm_zone *dst_zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) struct dmz_metadata *zmd = zrc->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) struct dm_io_region src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) sector_t block = 0, end_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) sector_t nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) sector_t src_zone_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) sector_t dst_zone_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) unsigned long flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (dmz_is_seq(src_zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) end_block = src_zone->wp_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) end_block = dmz_zone_nr_blocks(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) src_zone_block = dmz_start_block(zmd, src_zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) dst_zone_block = dmz_start_block(zmd, dst_zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (dmz_is_seq(dst_zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) set_bit(DM_KCOPYD_WRITE_SEQ, &flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) while (block < end_block) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) if (src_zone->dev->flags & DMZ_BDEV_DYING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (dst_zone->dev->flags & DMZ_BDEV_DYING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (dmz_reclaim_should_terminate(src_zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /* Get a valid region from the source zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) ret = dmz_first_valid_block(zmd, src_zone, &block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) nr_blocks = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * If we are writing in a sequential zone, we must make sure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * that writes are sequential. So Zeroout any eventual hole
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * between writes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (dmz_is_seq(dst_zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) ret = dmz_reclaim_align_wp(zrc, dst_zone, block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) src.bdev = src_zone->dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) src.sector = dmz_blk2sect(src_zone_block + block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) src.count = dmz_blk2sect(nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) dst.bdev = dst_zone->dev->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) dst.sector = dmz_blk2sect(dst_zone_block + block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) dst.count = src.count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* Copy the valid region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) set_bit(DMZ_RECLAIM_KCOPY, &zrc->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) dm_kcopyd_copy(zrc->kc, &src, 1, &dst, flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) dmz_reclaim_kcopy_end, zrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /* Wait for copy to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) wait_on_bit_io(&zrc->flags, DMZ_RECLAIM_KCOPY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) if (zrc->kc_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) return zrc->kc_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) block += nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) if (dmz_is_seq(dst_zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) dst_zone->wp_block = block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * Move valid blocks of dzone buffer zone into dzone (after its write pointer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * and free the buffer zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static int dmz_reclaim_buf(struct dmz_reclaim *zrc, struct dm_zone *dzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct dm_zone *bzone = dzone->bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) sector_t chunk_block = dzone->wp_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) struct dmz_metadata *zmd = zrc->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) DMDEBUG("(%s/%u): Chunk %u, move buf zone %u (weight %u) to data zone %u (weight %u)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) dmz_metadata_label(zmd), zrc->dev_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) dzone->chunk, bzone->id, dmz_weight(bzone),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) dzone->id, dmz_weight(dzone));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) /* Flush data zone into the buffer zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) ret = dmz_reclaim_copy(zrc, bzone, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) dmz_lock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /* Validate copied blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) ret = dmz_merge_valid_blocks(zmd, bzone, dzone, chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (ret == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) /* Free the buffer zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) dmz_invalidate_blocks(zmd, bzone, 0, dmz_zone_nr_blocks(zmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) dmz_unmap_zone(zmd, bzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) dmz_unlock_zone_reclaim(dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) dmz_free_zone(zmd, bzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) dmz_unlock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * Merge valid blocks of dzone into its buffer zone and free dzone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) static int dmz_reclaim_seq_data(struct dmz_reclaim *zrc, struct dm_zone *dzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) unsigned int chunk = dzone->chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) struct dm_zone *bzone = dzone->bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) struct dmz_metadata *zmd = zrc->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) DMDEBUG("(%s/%u): Chunk %u, move data zone %u (weight %u) to buf zone %u (weight %u)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) dmz_metadata_label(zmd), zrc->dev_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) chunk, dzone->id, dmz_weight(dzone),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) bzone->id, dmz_weight(bzone));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) /* Flush data zone into the buffer zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) ret = dmz_reclaim_copy(zrc, dzone, bzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) dmz_lock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /* Validate copied blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) ret = dmz_merge_valid_blocks(zmd, dzone, bzone, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (ret == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * Free the data zone and remap the chunk to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * the buffer zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) dmz_invalidate_blocks(zmd, dzone, 0, dmz_zone_nr_blocks(zmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) dmz_unmap_zone(zmd, bzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) dmz_unmap_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) dmz_unlock_zone_reclaim(dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) dmz_free_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) dmz_map_zone(zmd, bzone, chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) dmz_unlock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * Move valid blocks of the random data zone dzone into a free sequential zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * Once blocks are moved, remap the zone chunk to the sequential zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) static int dmz_reclaim_rnd_data(struct dmz_reclaim *zrc, struct dm_zone *dzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) unsigned int chunk = dzone->chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct dm_zone *szone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) struct dmz_metadata *zmd = zrc->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) int alloc_flags = DMZ_ALLOC_SEQ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) /* Get a free random or sequential zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) szone = dmz_alloc_zone(zmd, zrc->dev_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) alloc_flags | DMZ_ALLOC_RECLAIM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) if (!szone && alloc_flags == DMZ_ALLOC_SEQ && dmz_nr_cache_zones(zmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) alloc_flags = DMZ_ALLOC_RND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (!szone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) DMDEBUG("(%s/%u): Chunk %u, move %s zone %u (weight %u) to %s zone %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) dmz_metadata_label(zmd), zrc->dev_idx, chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) dmz_is_cache(dzone) ? "cache" : "rnd",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) dzone->id, dmz_weight(dzone),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) dmz_is_rnd(szone) ? "rnd" : "seq", szone->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /* Flush the random data zone into the sequential zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) ret = dmz_reclaim_copy(zrc, dzone, szone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) dmz_lock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (ret == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) /* Validate copied blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) ret = dmz_copy_valid_blocks(zmd, dzone, szone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) /* Free the sequential zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) dmz_free_zone(zmd, szone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /* Free the data zone and remap the chunk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) dmz_invalidate_blocks(zmd, dzone, 0, dmz_zone_nr_blocks(zmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) dmz_unmap_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) dmz_unlock_zone_reclaim(dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) dmz_free_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) dmz_map_zone(zmd, szone, chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) dmz_unlock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * Reclaim an empty zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) static void dmz_reclaim_empty(struct dmz_reclaim *zrc, struct dm_zone *dzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) struct dmz_metadata *zmd = zrc->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) dmz_lock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) dmz_unmap_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) dmz_unlock_zone_reclaim(dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) dmz_free_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) dmz_unlock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * Test if the target device is idle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static inline int dmz_target_idle(struct dmz_reclaim *zrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) return time_is_before_jiffies(zrc->atime + DMZ_IDLE_PERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * Find a candidate zone for reclaim and process it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static int dmz_do_reclaim(struct dmz_reclaim *zrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) struct dmz_metadata *zmd = zrc->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct dm_zone *dzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) struct dm_zone *rzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) unsigned long start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /* Get a data zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) dzone = dmz_get_zone_for_reclaim(zmd, zrc->dev_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) dmz_target_idle(zrc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (!dzone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) DMDEBUG("(%s/%u): No zone found to reclaim",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) dmz_metadata_label(zmd), zrc->dev_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) rzone = dzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) start = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (dmz_is_cache(dzone) || dmz_is_rnd(dzone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) if (!dmz_weight(dzone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /* Empty zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) dmz_reclaim_empty(zrc, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * Reclaim the random data zone by moving its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * valid data blocks to a free sequential zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) ret = dmz_reclaim_rnd_data(zrc, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) struct dm_zone *bzone = dzone->bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) sector_t chunk_block = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) ret = dmz_first_valid_block(zmd, bzone, &chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if (ret == 0 || chunk_block >= dzone->wp_block) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * The buffer zone is empty or its valid blocks are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * after the data zone write pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) ret = dmz_reclaim_buf(zrc, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) rzone = bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * Reclaim the data zone by merging it into the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * buffer zone so that the buffer zone itself can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * be later reclaimed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) ret = dmz_reclaim_seq_data(zrc, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (ret == -EINTR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) DMDEBUG("(%s/%u): reclaim zone %u interrupted",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) dmz_metadata_label(zmd), zrc->dev_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) rzone->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) DMDEBUG("(%s/%u): Failed to reclaim zone %u, err %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) dmz_metadata_label(zmd), zrc->dev_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) rzone->id, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) dmz_unlock_zone_reclaim(dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) ret = dmz_flush_metadata(zrc->metadata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) DMDEBUG("(%s/%u): Metadata flush for zone %u failed, err %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) dmz_metadata_label(zmd), zrc->dev_idx, rzone->id, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) DMDEBUG("(%s/%u): Reclaimed zone %u in %u ms",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) dmz_metadata_label(zmd), zrc->dev_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) rzone->id, jiffies_to_msecs(jiffies - start));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) static unsigned int dmz_reclaim_percentage(struct dmz_reclaim *zrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) struct dmz_metadata *zmd = zrc->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) unsigned int nr_cache = dmz_nr_cache_zones(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) unsigned int nr_unmap, nr_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (nr_cache) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) nr_zones = nr_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) nr_unmap = dmz_nr_unmap_cache_zones(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) nr_zones = dmz_nr_rnd_zones(zmd, zrc->dev_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) nr_unmap = dmz_nr_unmap_rnd_zones(zmd, zrc->dev_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (nr_unmap <= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return nr_unmap * 100 / nr_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * Test if reclaim is necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) static bool dmz_should_reclaim(struct dmz_reclaim *zrc, unsigned int p_unmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) unsigned int nr_reclaim;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) nr_reclaim = dmz_nr_rnd_zones(zrc->metadata, zrc->dev_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (dmz_nr_cache_zones(zrc->metadata)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * The first device in a multi-device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) * setup only contains cache zones, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * never start reclaim there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if (zrc->dev_idx == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) nr_reclaim += dmz_nr_cache_zones(zrc->metadata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) /* Reclaim when idle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (dmz_target_idle(zrc) && nr_reclaim)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /* If there are still plenty of cache zones, do not reclaim */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (p_unmap >= DMZ_RECLAIM_HIGH_UNMAP_ZONES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * If the percentage of unmapped cache zones is low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * reclaim even if the target is busy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) return p_unmap <= DMZ_RECLAIM_LOW_UNMAP_ZONES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * Reclaim work function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) static void dmz_reclaim_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) struct dmz_reclaim *zrc = container_of(work, struct dmz_reclaim, work.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) struct dmz_metadata *zmd = zrc->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) unsigned int p_unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (dmz_dev_is_dying(zmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) p_unmap = dmz_reclaim_percentage(zrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (!dmz_should_reclaim(zrc, p_unmap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) mod_delayed_work(zrc->wq, &zrc->work, DMZ_IDLE_PERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * We need to start reclaiming random zones: set up zone copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * throttling to either go fast if we are very low on random zones
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * and slower if there are still some free random zones to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) * as much as possible to negatively impact the user workload.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (dmz_target_idle(zrc) || p_unmap < DMZ_RECLAIM_LOW_UNMAP_ZONES / 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) /* Idle or very low percentage: go fast */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) zrc->kc_throttle.throttle = 100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) /* Busy but we still have some random zone: throttle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) zrc->kc_throttle.throttle = min(75U, 100U - p_unmap / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) DMDEBUG("(%s/%u): Reclaim (%u): %s, %u%% free zones (%u/%u cache %u/%u random)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) dmz_metadata_label(zmd), zrc->dev_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) zrc->kc_throttle.throttle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) (dmz_target_idle(zrc) ? "Idle" : "Busy"),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) p_unmap, dmz_nr_unmap_cache_zones(zmd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) dmz_nr_cache_zones(zmd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) dmz_nr_unmap_rnd_zones(zmd, zrc->dev_idx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) dmz_nr_rnd_zones(zmd, zrc->dev_idx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) ret = dmz_do_reclaim(zrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (ret && ret != -EINTR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (!dmz_check_dev(zmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) dmz_schedule_reclaim(zrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) * Initialize reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) int dmz_ctr_reclaim(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) struct dmz_reclaim **reclaim, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) struct dmz_reclaim *zrc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) zrc = kzalloc(sizeof(struct dmz_reclaim), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (!zrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) zrc->metadata = zmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) zrc->atime = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) zrc->dev_idx = idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) /* Reclaim kcopyd client */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) zrc->kc = dm_kcopyd_client_create(&zrc->kc_throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (IS_ERR(zrc->kc)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) ret = PTR_ERR(zrc->kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) zrc->kc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) /* Reclaim work */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) INIT_DELAYED_WORK(&zrc->work, dmz_reclaim_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) zrc->wq = alloc_ordered_workqueue("dmz_rwq_%s_%d", WQ_MEM_RECLAIM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) dmz_metadata_label(zmd), idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) if (!zrc->wq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) *reclaim = zrc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) queue_delayed_work(zrc->wq, &zrc->work, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (zrc->kc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) dm_kcopyd_client_destroy(zrc->kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) kfree(zrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) * Terminate reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) void dmz_dtr_reclaim(struct dmz_reclaim *zrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) cancel_delayed_work_sync(&zrc->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) destroy_workqueue(zrc->wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) dm_kcopyd_client_destroy(zrc->kc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) kfree(zrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) * Suspend reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) void dmz_suspend_reclaim(struct dmz_reclaim *zrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) cancel_delayed_work_sync(&zrc->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * Resume reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) void dmz_resume_reclaim(struct dmz_reclaim *zrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) queue_delayed_work(zrc->wq, &zrc->work, DMZ_IDLE_PERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * BIO accounting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) void dmz_reclaim_bio_acc(struct dmz_reclaim *zrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) zrc->atime = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * Start reclaim if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) void dmz_schedule_reclaim(struct dmz_reclaim *zrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) unsigned int p_unmap = dmz_reclaim_percentage(zrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) if (dmz_should_reclaim(zrc, p_unmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) mod_delayed_work(zrc->wq, &zrc->work, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags