^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2017 Western Digital Corporation or its affiliates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This file is released under the GPL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "dm-zoned.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/crc32.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #define DM_MSG_PREFIX "zoned metadata"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * Metadata version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #define DMZ_META_VER 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * On-disk super block magic.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define DMZ_MAGIC ((((unsigned int)('D')) << 24) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) (((unsigned int)('Z')) << 16) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) (((unsigned int)('B')) << 8) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) ((unsigned int)('D')))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * On disk super block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * This uses only 512 B but uses on disk a full 4KB block. This block is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * followed on disk by the mapping table of chunks to zones and the bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * blocks indicating zone block validity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * The overall resulting metadata format is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * (1) Super block (1 block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * (2) Chunk mapping table (nr_map_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * (3) Bitmap blocks (nr_bitmap_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * All metadata blocks are stored in conventional zones, starting from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * the first conventional zone found on disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) struct dmz_super {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* Magic number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) __le32 magic; /* 4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) /* Metadata version number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) __le32 version; /* 8 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) /* Generation number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) __le64 gen; /* 16 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /* This block number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) __le64 sb_block; /* 24 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /* The number of metadata blocks, including this super block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) __le32 nr_meta_blocks; /* 28 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) /* The number of sequential zones reserved for reclaim */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) __le32 nr_reserved_seq; /* 32 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /* The number of entries in the mapping table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) __le32 nr_chunks; /* 36 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* The number of blocks used for the chunk mapping table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) __le32 nr_map_blocks; /* 40 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /* The number of blocks used for the block bitmaps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) __le32 nr_bitmap_blocks; /* 44 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /* Checksum */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) __le32 crc; /* 48 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /* DM-Zoned label */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) u8 dmz_label[32]; /* 80 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) /* DM-Zoned UUID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) u8 dmz_uuid[16]; /* 96 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) /* Device UUID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) u8 dev_uuid[16]; /* 112 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) /* Padding to full 512B sector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) u8 reserved[400]; /* 512 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * Chunk mapping entry: entries are indexed by chunk number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * and give the zone ID (dzone_id) mapping the chunk on disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * This zone may be sequential or random. If it is a sequential
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * zone, a second zone (bzone_id) used as a write buffer may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * also be specified. This second zone will always be a randomly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * writeable zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) struct dmz_map {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) __le32 dzone_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) __le32 bzone_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * Chunk mapping table metadata: 512 8-bytes entries per 4KB block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define DMZ_MAP_ENTRIES (DMZ_BLOCK_SIZE / sizeof(struct dmz_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define DMZ_MAP_ENTRIES_SHIFT (ilog2(DMZ_MAP_ENTRIES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define DMZ_MAP_ENTRIES_MASK (DMZ_MAP_ENTRIES - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #define DMZ_MAP_UNMAPPED UINT_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * Meta data block descriptor (for cached metadata blocks).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) struct dmz_mblock {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) struct rb_node node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct list_head link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) sector_t no;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) unsigned int ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) unsigned long state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) void *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * Metadata block state flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) DMZ_META_DIRTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) DMZ_META_READING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) DMZ_META_WRITING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) DMZ_META_ERROR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * Super block information (one per metadata set).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct dmz_sb {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) sector_t block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) struct dmz_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct dmz_super *sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct dm_zone *zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * In-memory metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct dmz_metadata {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) struct dmz_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) unsigned int nr_devs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) char devname[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) char label[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) uuid_t uuid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) sector_t zone_bitmap_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) unsigned int zone_nr_bitmap_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) unsigned int zone_bits_per_mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) sector_t zone_nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) sector_t zone_nr_blocks_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) sector_t zone_nr_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) sector_t zone_nr_sectors_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) unsigned int nr_bitmap_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) unsigned int nr_map_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) unsigned int nr_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) unsigned int nr_useable_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) unsigned int nr_meta_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) unsigned int nr_meta_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) unsigned int nr_data_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) unsigned int nr_cache_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) unsigned int nr_rnd_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) unsigned int nr_reserved_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) unsigned int nr_chunks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /* Zone information array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct xarray zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) struct dmz_sb sb[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) unsigned int mblk_primary;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) unsigned int sb_version;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) u64 sb_gen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) unsigned int min_nr_mblks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) unsigned int max_nr_mblks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) atomic_t nr_mblks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct rw_semaphore mblk_sem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct mutex mblk_flush_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) spinlock_t mblk_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct rb_root mblk_rbtree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) struct list_head mblk_lru_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct list_head mblk_dirty_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) struct shrinker mblk_shrinker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /* Zone allocation management */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct mutex map_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) struct dmz_mblock **map_mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) unsigned int nr_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) atomic_t unmap_nr_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct list_head unmap_cache_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) struct list_head map_cache_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) atomic_t nr_reserved_seq_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) struct list_head reserved_seq_zones_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) wait_queue_head_t free_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) #define dmz_zmd_info(zmd, format, args...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) DMINFO("(%s): " format, (zmd)->label, ## args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #define dmz_zmd_err(zmd, format, args...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) DMERR("(%s): " format, (zmd)->label, ## args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #define dmz_zmd_warn(zmd, format, args...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) DMWARN("(%s): " format, (zmd)->label, ## args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) #define dmz_zmd_debug(zmd, format, args...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) DMDEBUG("(%s): " format, (zmd)->label, ## args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * Various accessors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (WARN_ON(!zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return zone->id - zone->dev->zone_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return (sector_t)zone_id << zmd->zone_nr_sectors_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return (sector_t)zone_id << zmd->zone_nr_blocks_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) return zmd->zone_nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) unsigned int dmz_zone_nr_blocks_shift(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return zmd->zone_nr_blocks_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) return zmd->zone_nr_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) return zmd->zone_nr_sectors_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) unsigned int dmz_nr_zones(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return zmd->nr_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) return zmd->nr_chunks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return zmd->dev[idx].nr_rnd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return atomic_read(&zmd->dev[idx].unmap_nr_rnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) return zmd->nr_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return atomic_read(&zmd->unmap_nr_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return zmd->dev[idx].nr_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) return atomic_read(&zmd->dev[idx].unmap_nr_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return xa_load(&zmd->zones, zone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) static struct dm_zone *dmz_insert(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) unsigned int zone_id, struct dmz_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) struct dm_zone *zone = kzalloc(sizeof(struct dm_zone), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (!zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (xa_insert(&zmd->zones, zone_id, zone, GFP_KERNEL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) kfree(zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return ERR_PTR(-EBUSY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) INIT_LIST_HEAD(&zone->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) atomic_set(&zone->refcount, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) zone->id = zone_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) zone->chunk = DMZ_MAP_UNMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) zone->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) return zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) const char *dmz_metadata_label(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return (const char *)zmd->label;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) bool dmz_check_dev(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) for (i = 0; i < zmd->nr_devs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (!dmz_check_bdev(&zmd->dev[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) bool dmz_dev_is_dying(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) for (i = 0; i < zmd->nr_devs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (dmz_bdev_is_dying(&zmd->dev[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * Lock/unlock mapping table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * The map lock also protects all the zone lists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) void dmz_lock_map(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) mutex_lock(&zmd->map_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) void dmz_unlock_map(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) mutex_unlock(&zmd->map_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * Lock/unlock metadata access. This is a "read" lock on a semaphore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * that prevents metadata flush from running while metadata are being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * modified. The actual metadata write mutual exclusion is achieved with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * the map lock and zone state management (active and reclaim state are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * mutually exclusive).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) void dmz_lock_metadata(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) down_read(&zmd->mblk_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) void dmz_unlock_metadata(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) up_read(&zmd->mblk_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * Lock/unlock flush: prevent concurrent executions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * of dmz_flush_metadata as well as metadata modification in reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * while flush is being executed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) void dmz_lock_flush(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) mutex_lock(&zmd->mblk_flush_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) void dmz_unlock_flush(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) mutex_unlock(&zmd->mblk_flush_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * Allocate a metadata block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) sector_t mblk_no)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) struct dmz_mblock *mblk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) /* See if we can reuse cached blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (zmd->max_nr_mblks && atomic_read(&zmd->nr_mblks) > zmd->max_nr_mblks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) spin_lock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) mblk = list_first_entry_or_null(&zmd->mblk_lru_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct dmz_mblock, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (mblk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) list_del_init(&mblk->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) rb_erase(&mblk->node, &zmd->mblk_rbtree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) mblk->no = mblk_no;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) return mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) /* Allocate a new block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) mblk = kmalloc(sizeof(struct dmz_mblock), GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (!mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) mblk->page = alloc_page(GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if (!mblk->page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) kfree(mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) RB_CLEAR_NODE(&mblk->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) INIT_LIST_HEAD(&mblk->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) mblk->ref = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) mblk->state = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) mblk->no = mblk_no;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) mblk->data = page_address(mblk->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) atomic_inc(&zmd->nr_mblks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) return mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * Free a metadata block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) __free_pages(mblk->page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) kfree(mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) atomic_dec(&zmd->nr_mblks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * Insert a metadata block in the rbtree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) struct rb_root *root = &zmd->mblk_rbtree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) struct rb_node **new = &(root->rb_node), *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) struct dmz_mblock *b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) /* Figure out where to put the new node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) while (*new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) b = container_of(*new, struct dmz_mblock, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) parent = *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) new = (b->no < mblk->no) ? &((*new)->rb_left) : &((*new)->rb_right);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) /* Add new node and rebalance tree */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) rb_link_node(&mblk->node, parent, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) rb_insert_color(&mblk->node, root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * Lookup a metadata block in the rbtree. If the block is found, increment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * its reference count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) sector_t mblk_no)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct rb_root *root = &zmd->mblk_rbtree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) struct rb_node *node = root->rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) while (node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) mblk = container_of(node, struct dmz_mblock, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (mblk->no == mblk_no) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) * If this is the first reference to the block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * remove it from the LRU list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) mblk->ref++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (mblk->ref == 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) !test_bit(DMZ_META_DIRTY, &mblk->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) list_del_init(&mblk->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) node = (mblk->no < mblk_no) ? node->rb_left : node->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * Metadata block BIO end callback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) static void dmz_mblock_bio_end_io(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) struct dmz_mblock *mblk = bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) int flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) if (bio->bi_status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) set_bit(DMZ_META_ERROR, &mblk->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (bio_op(bio) == REQ_OP_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) flag = DMZ_META_WRITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) flag = DMZ_META_READING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) clear_bit_unlock(flag, &mblk->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) wake_up_bit(&mblk->state, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * Read an uncached metadata block from disk and add it to the cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) sector_t mblk_no)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) struct dmz_mblock *mblk, *m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (dmz_bdev_is_dying(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) return ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) /* Get a new block and a BIO to read it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) mblk = dmz_alloc_mblock(zmd, mblk_no);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (!mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) bio = bio_alloc(GFP_NOIO, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) if (!bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) dmz_free_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) spin_lock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * Make sure that another context did not start reading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * the block already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) m = dmz_get_mblock_fast(zmd, mblk_no);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (m) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) dmz_free_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) return m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) mblk->ref++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) set_bit(DMZ_META_READING, &mblk->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) dmz_insert_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) /* Submit read BIO */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) bio->bi_iter.bi_sector = dmz_blk2sect(block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) bio_set_dev(bio, dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) bio->bi_private = mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) bio->bi_end_io = dmz_mblock_bio_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) bio_set_op_attrs(bio, REQ_OP_READ, REQ_META | REQ_PRIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) submit_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * Free metadata blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) unsigned long limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) unsigned long count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (!zmd->max_nr_mblks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) while (!list_empty(&zmd->mblk_lru_list) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) atomic_read(&zmd->nr_mblks) > zmd->min_nr_mblks &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) count < limit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) mblk = list_first_entry(&zmd->mblk_lru_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) struct dmz_mblock, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) list_del_init(&mblk->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) rb_erase(&mblk->node, &zmd->mblk_rbtree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) dmz_free_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) * For mblock shrinker: get the number of unused metadata blocks in the cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) static unsigned long dmz_mblock_shrinker_count(struct shrinker *shrink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) return atomic_read(&zmd->nr_mblks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * For mblock shrinker: scan unused metadata blocks and shrink the cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) static unsigned long dmz_mblock_shrinker_scan(struct shrinker *shrink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) unsigned long count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) spin_lock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) count = dmz_shrink_mblock_cache(zmd, sc->nr_to_scan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) return count ? count : SHRINK_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * Release a metadata block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) static void dmz_release_mblock(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) struct dmz_mblock *mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (!mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) spin_lock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) mblk->ref--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) if (mblk->ref == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (test_bit(DMZ_META_ERROR, &mblk->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) rb_erase(&mblk->node, &zmd->mblk_rbtree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) dmz_free_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) } else if (!test_bit(DMZ_META_DIRTY, &mblk->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) list_add_tail(&mblk->link, &zmd->mblk_lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) dmz_shrink_mblock_cache(zmd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * Get a metadata block from the rbtree. If the block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * is not present, read it from disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) sector_t mblk_no)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) /* Check rbtree */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) spin_lock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) mblk = dmz_get_mblock_fast(zmd, mblk_no);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (!mblk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) /* Cache miss: read the block from disk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) mblk = dmz_get_mblock_slow(zmd, mblk_no);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) if (IS_ERR(mblk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) return mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) /* Wait for on-going read I/O and check for error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) wait_on_bit_io(&mblk->state, DMZ_META_READING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) if (test_bit(DMZ_META_ERROR, &mblk->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) dmz_release_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) dmz_check_bdev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) return ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) return mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * Mark a metadata block dirty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) spin_lock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (!test_and_set_bit(DMZ_META_DIRTY, &mblk->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) list_add_tail(&mblk->link, &zmd->mblk_dirty_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) * Issue a metadata block write BIO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) unsigned int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) struct dmz_dev *dev = zmd->sb[set].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) sector_t block = zmd->sb[set].block + mblk->no;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (dmz_bdev_is_dying(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) bio = bio_alloc(GFP_NOIO, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (!bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) set_bit(DMZ_META_ERROR, &mblk->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) set_bit(DMZ_META_WRITING, &mblk->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) bio->bi_iter.bi_sector = dmz_blk2sect(block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) bio_set_dev(bio, dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) bio->bi_private = mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) bio->bi_end_io = dmz_mblock_bio_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_META | REQ_PRIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) submit_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * Read/write a metadata block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) static int dmz_rdwr_block(struct dmz_dev *dev, int op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) sector_t block, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) if (WARN_ON(!dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) if (dmz_bdev_is_dying(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) bio = bio_alloc(GFP_NOIO, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (!bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) bio->bi_iter.bi_sector = dmz_blk2sect(block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) bio_set_dev(bio, dev->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) bio_set_op_attrs(bio, op, REQ_SYNC | REQ_META | REQ_PRIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) bio_add_page(bio, page, DMZ_BLOCK_SIZE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) ret = submit_bio_wait(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) dmz_check_bdev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * Write super block of the specified metadata set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) struct dmz_mblock *mblk = zmd->sb[set].mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) struct dmz_super *sb = zmd->sb[set].sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) struct dmz_dev *dev = zmd->sb[set].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) sector_t sb_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) u64 sb_gen = zmd->sb_gen + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) sb->magic = cpu_to_le32(DMZ_MAGIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) sb->version = cpu_to_le32(zmd->sb_version);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (zmd->sb_version > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) BUILD_BUG_ON(UUID_SIZE != 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) export_uuid(sb->dmz_uuid, &zmd->uuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) export_uuid(sb->dev_uuid, &dev->uuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) sb->gen = cpu_to_le64(sb_gen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * The metadata always references the absolute block address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * ie relative to the entire block range, not the per-device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * block address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) sb->sb_block = cpu_to_le64(sb_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) sb->crc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) sb->crc = cpu_to_le32(crc32_le(sb_gen, (unsigned char *)sb, DMZ_BLOCK_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) mblk->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) * Write dirty metadata blocks to the specified set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) struct list_head *write_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) unsigned int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) struct dmz_dev *dev = zmd->sb[set].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) int ret = 0, nr_mblks_submitted = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) /* Issue writes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) list_for_each_entry(mblk, write_list, link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) ret = dmz_write_mblock(zmd, mblk, set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) nr_mblks_submitted++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) /* Wait for completion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) list_for_each_entry(mblk, write_list, link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) if (!nr_mblks_submitted)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) wait_on_bit_io(&mblk->state, DMZ_META_WRITING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) if (test_bit(DMZ_META_ERROR, &mblk->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) clear_bit(DMZ_META_ERROR, &mblk->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) dmz_check_bdev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) nr_mblks_submitted--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) /* Flush drive cache (this will also sync data) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * Log dirty metadata blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) struct list_head *write_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) unsigned int log_set = zmd->mblk_primary ^ 0x1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) /* Write dirty blocks to the log */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) ret = dmz_write_dirty_mblocks(zmd, write_list, log_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) * No error so far: now validate the log by updating the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * log index super block generation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) ret = dmz_write_sb(zmd, log_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * Flush dirty metadata blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) int dmz_flush_metadata(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) struct list_head write_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) struct dmz_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) if (WARN_ON(!zmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) INIT_LIST_HEAD(&write_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * Make sure that metadata blocks are stable before logging: take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * the write lock on the metadata semaphore to prevent target BIOs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) * from modifying metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) down_write(&zmd->mblk_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) dev = zmd->sb[zmd->mblk_primary].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) * This is called from the target flush work and reclaim work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) * Concurrent execution is not allowed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) dmz_lock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) if (dmz_bdev_is_dying(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) /* Get dirty blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) spin_lock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) list_splice_init(&zmd->mblk_dirty_list, &write_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) /* If there are no dirty metadata blocks, just flush the device cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) if (list_empty(&write_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * The primary metadata set is still clean. Keep it this way until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) * all updates are successful in the secondary set. That is, use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * the secondary set as a log.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) ret = dmz_log_dirty_mblocks(zmd, &write_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * The log is on disk. It is now safe to update in place
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) * in the primary metadata set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) ret = dmz_write_dirty_mblocks(zmd, &write_list, zmd->mblk_primary);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) ret = dmz_write_sb(zmd, zmd->mblk_primary);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) while (!list_empty(&write_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) mblk = list_first_entry(&write_list, struct dmz_mblock, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) list_del_init(&mblk->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) spin_lock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) clear_bit(DMZ_META_DIRTY, &mblk->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) if (mblk->ref == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) list_add_tail(&mblk->link, &zmd->mblk_lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) zmd->sb_gen++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) dmz_unlock_flush(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) up_write(&zmd->mblk_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) if (!list_empty(&write_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) spin_lock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) list_splice(&write_list, &zmd->mblk_dirty_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) spin_unlock(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) if (!dmz_check_bdev(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) * Check super block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) bool tertiary)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) struct dmz_super *sb = dsb->sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) struct dmz_dev *dev = dsb->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) unsigned int nr_meta_zones, nr_data_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) u32 crc, stored_crc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) u64 gen, sb_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) if (le32_to_cpu(sb->magic) != DMZ_MAGIC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) dmz_dev_err(dev, "Invalid meta magic (needed 0x%08x, got 0x%08x)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) DMZ_MAGIC, le32_to_cpu(sb->magic));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) zmd->sb_version = le32_to_cpu(sb->version);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) if (zmd->sb_version > DMZ_META_VER) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) dmz_dev_err(dev, "Invalid meta version (needed %d, got %d)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) DMZ_META_VER, zmd->sb_version);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) if (zmd->sb_version < 2 && tertiary) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) dmz_dev_err(dev, "Tertiary superblocks are not supported");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) gen = le64_to_cpu(sb->gen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) stored_crc = le32_to_cpu(sb->crc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) sb->crc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) crc = crc32_le(gen, (unsigned char *)sb, DMZ_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) if (crc != stored_crc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) dmz_dev_err(dev, "Invalid checksum (needed 0x%08x, got 0x%08x)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) crc, stored_crc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) sb_block = le64_to_cpu(sb->sb_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) dmz_dev_err(dev, "Invalid superblock position "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) "(is %llu expected %llu)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) sb_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) (u64)dsb->zone->id << zmd->zone_nr_blocks_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) if (zmd->sb_version > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) uuid_t sb_uuid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) import_uuid(&sb_uuid, sb->dmz_uuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) if (uuid_is_null(&sb_uuid)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) dmz_dev_err(dev, "NULL DM-Zoned uuid");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) } else if (uuid_is_null(&zmd->uuid)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) uuid_copy(&zmd->uuid, &sb_uuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) } else if (!uuid_equal(&zmd->uuid, &sb_uuid)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) dmz_dev_err(dev, "mismatching DM-Zoned uuid, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) "is %pUl expected %pUl",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) &sb_uuid, &zmd->uuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) if (!strlen(zmd->label))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) else if (memcmp(zmd->label, sb->dmz_label, BDEVNAME_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) dmz_dev_err(dev, "mismatching DM-Zoned label, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) "is %s expected %s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) sb->dmz_label, zmd->label);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) import_uuid(&dev->uuid, sb->dev_uuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) if (uuid_is_null(&dev->uuid)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) dmz_dev_err(dev, "NULL device uuid");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) if (tertiary) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) * Generation number should be 0, but it doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) * really matter if it isn't.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) if (gen != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) dmz_dev_warn(dev, "Invalid generation %llu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) gen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) >> zmd->zone_nr_blocks_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if (!nr_meta_zones ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) (zmd->nr_devs <= 1 && nr_meta_zones >= zmd->nr_rnd_zones) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) (zmd->nr_devs > 1 && nr_meta_zones >= zmd->nr_cache_zones)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) dmz_dev_err(dev, "Invalid number of metadata blocks");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) if (!le32_to_cpu(sb->nr_reserved_seq) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) dmz_dev_err(dev, "Invalid number of reserved sequential zones");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) nr_data_zones = zmd->nr_useable_zones -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) (nr_meta_zones * 2 + le32_to_cpu(sb->nr_reserved_seq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) if (le32_to_cpu(sb->nr_chunks) > nr_data_zones) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) dmz_dev_err(dev, "Invalid number of chunks %u / %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) le32_to_cpu(sb->nr_chunks), nr_data_zones);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) /* OK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) zmd->nr_chunks = le32_to_cpu(sb->nr_chunks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) zmd->nr_meta_zones = nr_meta_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) zmd->nr_data_zones = nr_data_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) * Read the first or second super block from disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) dmz_zmd_debug(zmd, "read superblock set %d dev %s block %llu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) set, sb->dev->name, sb->block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) return dmz_rdwr_block(sb->dev, REQ_OP_READ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) sb->block, sb->mblk->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) * Determine the position of the secondary super blocks on disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) * This is used only if a corruption of the primary super block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) * is detected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) unsigned int zone_id = zmd->sb[0].zone->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) /* Allocate a block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) mblk = dmz_alloc_mblock(zmd, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) if (!mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) zmd->sb[1].mblk = mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) zmd->sb[1].sb = mblk->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) /* Bad first super block: search for the second one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) zmd->sb[1].zone = dmz_get(zmd, zone_id + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) zmd->sb[1].dev = zmd->sb[0].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) for (i = 1; i < zmd->nr_rnd_zones; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) if (dmz_read_sb(zmd, &zmd->sb[1], 1) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) zmd->sb[1].block += zone_nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) zmd->sb[1].zone = dmz_get(zmd, zone_id + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) dmz_free_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) zmd->sb[1].mblk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) zmd->sb[1].zone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) zmd->sb[1].dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) * Read a super block from disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) /* Allocate a block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) mblk = dmz_alloc_mblock(zmd, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) if (!mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) sb->mblk = mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) sb->sb = mblk->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) /* Read super block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) ret = dmz_read_sb(zmd, sb, set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) dmz_free_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) sb->mblk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) * Recover a metadata set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) unsigned int src_set = dst_set ^ 0x1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) dmz_dev_warn(zmd->sb[dst_set].dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) "Metadata set %u invalid: recovering", dst_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) if (dst_set == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) page = alloc_page(GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) /* Copy metadata blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) for (i = 1; i < zmd->nr_meta_blocks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) ret = dmz_rdwr_block(zmd->sb[src_set].dev, REQ_OP_READ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) zmd->sb[src_set].block + i, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) ret = dmz_rdwr_block(zmd->sb[dst_set].dev, REQ_OP_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) zmd->sb[dst_set].block + i, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) /* Finalize with the super block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) if (!zmd->sb[dst_set].mblk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) if (!zmd->sb[dst_set].mblk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) ret = dmz_write_sb(zmd, dst_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) __free_pages(page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) * Get super block from disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) static int dmz_load_sb(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) bool sb_good[2] = {false, false};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) u64 sb_gen[2] = {0, 0};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) if (!zmd->sb[0].zone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) dmz_zmd_err(zmd, "Primary super block zone not set");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) /* Read and check the primary super block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) zmd->sb[0].dev = zmd->sb[0].zone->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) ret = dmz_get_sb(zmd, &zmd->sb[0], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) ret = dmz_check_sb(zmd, &zmd->sb[0], false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) /* Read and check secondary super block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) if (ret == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) sb_good[0] = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) if (!zmd->sb[1].zone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) unsigned int zone_id =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) zmd->sb[0].zone->id + zmd->nr_meta_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) zmd->sb[1].zone = dmz_get(zmd, zone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) zmd->sb[1].dev = zmd->sb[0].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) ret = dmz_get_sb(zmd, &zmd->sb[1], 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) ret = dmz_lookup_secondary_sb(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) ret = dmz_check_sb(zmd, &zmd->sb[1], false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) sb_good[1] = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) /* Use highest generation sb first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) if (!sb_good[0] && !sb_good[1]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) dmz_zmd_err(zmd, "No valid super block found");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) if (sb_good[0])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) ret = dmz_recover_mblocks(zmd, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) dmz_dev_err(zmd->sb[0].dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) "Recovery of superblock 0 failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) if (sb_good[1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) ret = dmz_recover_mblocks(zmd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) dmz_dev_err(zmd->sb[1].dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) "Recovery of superblock 1 failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) if (sb_gen[0] >= sb_gen[1]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) zmd->sb_gen = sb_gen[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) zmd->mblk_primary = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) zmd->sb_gen = sb_gen[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) zmd->mblk_primary = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) "Using super block %u (gen %llu)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) zmd->mblk_primary, zmd->sb_gen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) if (zmd->sb_version > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) struct dmz_sb *sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) sb = kzalloc(sizeof(struct dmz_sb), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) if (!sb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) for (i = 1; i < zmd->nr_devs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) sb->block = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) sb->zone = dmz_get(zmd, zmd->dev[i].zone_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) sb->dev = &zmd->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) if (!dmz_is_meta(sb->zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) dmz_dev_err(sb->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) "Tertiary super block zone %u not marked as metadata zone",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) sb->zone->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) goto out_kfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) ret = dmz_get_sb(zmd, sb, i + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) dmz_dev_err(sb->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) "Read tertiary super block failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) dmz_free_mblock(zmd, sb->mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) goto out_kfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) ret = dmz_check_sb(zmd, sb, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) dmz_free_mblock(zmd, sb->mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) if (ret == -EINVAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) goto out_kfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) out_kfree:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) kfree(sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) * Initialize a zone descriptor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) static int dmz_init_zone(struct blk_zone *blkz, unsigned int num, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) struct dmz_dev *dev = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) struct dmz_metadata *zmd = dev->metadata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) int idx = num + dev->zone_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) struct dm_zone *zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) zone = dmz_insert(zmd, idx, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) if (IS_ERR(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) return PTR_ERR(zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) if (blkz->len != zmd->zone_nr_sectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) if (zmd->sb_version > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) /* Ignore the eventual runt (smaller) zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) set_bit(DMZ_OFFLINE, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) } else if (blkz->start + blkz->len == dev->capacity)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) * Devices that have zones with a capacity smaller than the zone size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) * (e.g. NVMe zoned namespaces) are not supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) if (blkz->capacity != blkz->len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) switch (blkz->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) case BLK_ZONE_TYPE_CONVENTIONAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) set_bit(DMZ_RND, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) case BLK_ZONE_TYPE_SEQWRITE_REQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) case BLK_ZONE_TYPE_SEQWRITE_PREF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) set_bit(DMZ_SEQ, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) if (dmz_is_rnd(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) zone->wp_block = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) if (blkz->cond == BLK_ZONE_COND_OFFLINE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) set_bit(DMZ_OFFLINE, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) else if (blkz->cond == BLK_ZONE_COND_READONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) set_bit(DMZ_READ_ONLY, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) zmd->nr_useable_zones++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) if (dmz_is_rnd(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) zmd->nr_rnd_zones++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) if (zmd->nr_devs == 1 && !zmd->sb[0].zone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) /* Primary super block zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) zmd->sb[0].zone = zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) if (zmd->nr_devs > 1 && num == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) * Tertiary superblock zones are always at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) * start of the zoned devices, so mark them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) * as metadata zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) set_bit(DMZ_META, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) sector_t zone_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) for(idx = 0; idx < dev->nr_zones; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) struct dm_zone *zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) zone = dmz_insert(zmd, idx, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) if (IS_ERR(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) return PTR_ERR(zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) set_bit(DMZ_CACHE, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) zone->wp_block = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) zmd->nr_cache_zones++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) zmd->nr_useable_zones++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) if (dev->capacity - zone_offset < zmd->zone_nr_sectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) /* Disable runt zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) set_bit(DMZ_OFFLINE, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) zone_offset += zmd->zone_nr_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) * Free zones descriptors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) static void dmz_drop_zones(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) for(idx = 0; idx < zmd->nr_zones; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) struct dm_zone *zone = xa_load(&zmd->zones, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) kfree(zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) xa_erase(&zmd->zones, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) xa_destroy(&zmd->zones);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) * Allocate and initialize zone descriptors using the zone
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) * information from disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) static int dmz_init_zones(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) struct dmz_dev *zoned_dev = &zmd->dev[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) /* Init */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) zmd->zone_nr_bitmap_blocks =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) DMZ_BLOCK_SIZE_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) /* Allocate zone array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) zmd->nr_zones = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) for (i = 0; i < zmd->nr_devs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) struct dmz_dev *dev = &zmd->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) dev->metadata = zmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) zmd->nr_zones += dev->nr_zones;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) atomic_set(&dev->unmap_nr_rnd, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) INIT_LIST_HEAD(&dev->unmap_rnd_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) INIT_LIST_HEAD(&dev->map_rnd_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) atomic_set(&dev->unmap_nr_seq, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) INIT_LIST_HEAD(&dev->unmap_seq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) INIT_LIST_HEAD(&dev->map_seq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) if (!zmd->nr_zones) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) DMERR("(%s): No zones found", zmd->devname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) xa_init(&zmd->zones);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) DMDEBUG("(%s): Using %zu B for zone information",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) if (zmd->nr_devs > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) ret = dmz_emulate_zones(zmd, &zmd->dev[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) DMDEBUG("(%s): Failed to emulate zones, error %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) zmd->devname, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) dmz_drop_zones(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) * Primary superblock zone is always at zone 0 when multiple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) * drives are present.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) zmd->sb[0].zone = dmz_get(zmd, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) for (i = 1; i < zmd->nr_devs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) zoned_dev = &zmd->dev[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) ret = blkdev_report_zones(zoned_dev->bdev, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) BLK_ALL_ZONES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) dmz_init_zone, zoned_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) DMDEBUG("(%s): Failed to report zones, error %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) zmd->devname, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) dmz_drop_zones(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) * Get zone information and initialize zone descriptors. At the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) * time, determine where the super block should be: first block of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) * first randomly writable zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) ret = blkdev_report_zones(zoned_dev->bdev, 0, BLK_ALL_ZONES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) dmz_init_zone, zoned_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) DMDEBUG("(%s): Failed to report zones, error %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) zmd->devname, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) dmz_drop_zones(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) static int dmz_update_zone_cb(struct blk_zone *blkz, unsigned int idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) struct dm_zone *zone = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) clear_bit(DMZ_OFFLINE, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) clear_bit(DMZ_READ_ONLY, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) if (blkz->cond == BLK_ZONE_COND_OFFLINE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) set_bit(DMZ_OFFLINE, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) else if (blkz->cond == BLK_ZONE_COND_READONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) set_bit(DMZ_READ_ONLY, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) if (dmz_is_seq(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) zone->wp_block = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) * Update a zone information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) struct dmz_dev *dev = zone->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) unsigned int noio_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) if (dev->flags & DMZ_BDEV_REGULAR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) * Get zone information from disk. Since blkdev_report_zones() uses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) * GFP_KERNEL by default for memory allocations, set the per-task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) * PF_MEMALLOC_NOIO flag so that all allocations are done as if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) * GFP_NOIO was specified.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) noio_flag = memalloc_noio_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) ret = blkdev_report_zones(dev->bdev, dmz_start_sect(zmd, zone), 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) dmz_update_zone_cb, zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) memalloc_noio_restore(noio_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) dmz_dev_err(dev, "Get zone %u report failed",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) zone->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) dmz_check_bdev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) * Check a zone write pointer position when the zone is marked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) * with the sequential write error flag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) struct dmz_dev *dev = zone->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) unsigned int wp = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) wp = zone->wp_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) ret = dmz_update_zone(zmd, zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) dmz_dev_warn(dev, "Processing zone %u write error (zone wp %u/%u)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) zone->id, zone->wp_block, wp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) if (zone->wp_block < wp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) dmz_invalidate_blocks(zmd, zone, zone->wp_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) wp - zone->wp_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) * Reset a zone write pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) * Ignore offline zones, read only zones,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) * and conventional zones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) if (dmz_is_offline(zone) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) dmz_is_readonly(zone) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) dmz_is_rnd(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) if (!dmz_is_empty(zone) || dmz_seq_write_err(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) struct dmz_dev *dev = zone->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) ret = blkdev_zone_mgmt(dev->bdev, REQ_OP_ZONE_RESET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) dmz_start_sect(zmd, zone),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) zmd->zone_nr_sectors, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) dmz_dev_err(dev, "Reset zone %u failed %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) zone->id, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) /* Clear write error bit and rewind write pointer position */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) clear_bit(DMZ_SEQ_WRITE_ERR, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) zone->wp_block = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) * Initialize chunk mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) static int dmz_load_mapping(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) struct dm_zone *dzone, *bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) struct dmz_mblock *dmap_mblk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) struct dmz_map *dmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) unsigned int i = 0, e = 0, chunk = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) unsigned int dzone_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) unsigned int bzone_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) /* Metadata block array for the chunk mapping table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) zmd->map_mblk = kcalloc(zmd->nr_map_blocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) sizeof(struct dmz_mblk *), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) if (!zmd->map_mblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) /* Get chunk mapping table blocks and initialize zone mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) while (chunk < zmd->nr_chunks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) if (!dmap_mblk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) /* Get mapping block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) dmap_mblk = dmz_get_mblock(zmd, i + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) if (IS_ERR(dmap_mblk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) return PTR_ERR(dmap_mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) zmd->map_mblk[i] = dmap_mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) dmap = (struct dmz_map *) dmap_mblk->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) e = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) /* Check data zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) dzone_id = le32_to_cpu(dmap[e].dzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) if (dzone_id == DMZ_MAP_UNMAPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) if (dzone_id >= zmd->nr_zones) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) dmz_zmd_err(zmd, "Chunk %u mapping: invalid data zone ID %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) chunk, dzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) dzone = dmz_get(zmd, dzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) if (!dzone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) dmz_zmd_err(zmd, "Chunk %u mapping: data zone %u not present",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) chunk, dzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) set_bit(DMZ_DATA, &dzone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) dzone->chunk = chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) dmz_get_zone_weight(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) if (dmz_is_cache(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) list_add_tail(&dzone->link, &zmd->map_cache_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) else if (dmz_is_rnd(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) list_add_tail(&dzone->link, &dzone->dev->map_rnd_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) list_add_tail(&dzone->link, &dzone->dev->map_seq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) /* Check buffer zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) bzone_id = le32_to_cpu(dmap[e].bzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) if (bzone_id == DMZ_MAP_UNMAPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) if (bzone_id >= zmd->nr_zones) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone ID %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) chunk, bzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) bzone = dmz_get(zmd, bzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) if (!bzone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) dmz_zmd_err(zmd, "Chunk %u mapping: buffer zone %u not present",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) chunk, bzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) if (!dmz_is_rnd(bzone) && !dmz_is_cache(bzone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) chunk, bzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) set_bit(DMZ_DATA, &bzone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) set_bit(DMZ_BUF, &bzone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) bzone->chunk = chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) bzone->bzone = dzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) dzone->bzone = bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) dmz_get_zone_weight(zmd, bzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) if (dmz_is_cache(bzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) list_add_tail(&bzone->link, &zmd->map_cache_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) list_add_tail(&bzone->link, &bzone->dev->map_rnd_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) chunk++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) e++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) if (e >= DMZ_MAP_ENTRIES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) dmap_mblk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) * At this point, only meta zones and mapped data zones were
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) * fully initialized. All remaining zones are unmapped data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) * zones. Finish initializing those here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) for (i = 0; i < zmd->nr_zones; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) dzone = dmz_get(zmd, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) if (!dzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) if (dmz_is_meta(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) if (dmz_is_offline(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) if (dmz_is_cache(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) zmd->nr_cache++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) else if (dmz_is_rnd(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) dzone->dev->nr_rnd++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) dzone->dev->nr_seq++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) if (dmz_is_data(dzone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) /* Already initialized */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) /* Unmapped data zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) set_bit(DMZ_DATA, &dzone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) dzone->chunk = DMZ_MAP_UNMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) if (dmz_is_cache(dzone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) list_add_tail(&dzone->link, &zmd->unmap_cache_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) atomic_inc(&zmd->unmap_nr_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) } else if (dmz_is_rnd(dzone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) list_add_tail(&dzone->link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) &dzone->dev->unmap_rnd_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) atomic_inc(&dzone->dev->unmap_nr_rnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) } else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) set_bit(DMZ_RESERVED, &dzone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) atomic_inc(&zmd->nr_reserved_seq_zones);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) dzone->dev->nr_seq--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) list_add_tail(&dzone->link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) &dzone->dev->unmap_seq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) atomic_inc(&dzone->dev->unmap_nr_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) * Set a data chunk mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) unsigned int dzone_id, unsigned int bzone_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) struct dmz_map *dmap = (struct dmz_map *) dmap_mblk->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) int map_idx = chunk & DMZ_MAP_ENTRIES_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) dmap[map_idx].dzone_id = cpu_to_le32(dzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) dmap[map_idx].bzone_id = cpu_to_le32(bzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) dmz_dirty_mblock(zmd, dmap_mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) * The list of mapped zones is maintained in LRU order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) * This rotates a zone at the end of its map list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) if (list_empty(&zone->link))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) list_del_init(&zone->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) if (dmz_is_seq(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) /* LRU rotate sequential zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) list_add_tail(&zone->link, &zone->dev->map_seq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) } else if (dmz_is_cache(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) /* LRU rotate cache zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) list_add_tail(&zone->link, &zmd->map_cache_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) /* LRU rotate random zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) list_add_tail(&zone->link, &zone->dev->map_rnd_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) * The list of mapped random zones is maintained
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) * in LRU order. This rotates a zone at the end of the list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) __dmz_lru_zone(zmd, zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) if (zone->bzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) __dmz_lru_zone(zmd, zone->bzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) * Wait for any zone to be freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) static void dmz_wait_for_free_zones(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) DEFINE_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) prepare_to_wait(&zmd->free_wq, &wait, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) dmz_unlock_metadata(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) io_schedule_timeout(HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) dmz_lock_metadata(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) finish_wait(&zmd->free_wq, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) * Lock a zone for reclaim (set the zone RECLAIM bit).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) * Returns false if the zone cannot be locked or if it is already locked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) * and 1 otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) int dmz_lock_zone_reclaim(struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) /* Active zones cannot be reclaimed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) if (dmz_is_active(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) return !test_and_set_bit(DMZ_RECLAIM, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) * Clear a zone reclaim flag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) void dmz_unlock_zone_reclaim(struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) WARN_ON(dmz_is_active(zone));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) WARN_ON(!dmz_in_reclaim(zone));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) clear_bit_unlock(DMZ_RECLAIM, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) wake_up_bit(&zone->flags, DMZ_RECLAIM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) * Wait for a zone reclaim to complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) dmz_unlock_metadata(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) set_bit(DMZ_RECLAIM_TERMINATE, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) wait_on_bit_timeout(&zone->flags, DMZ_RECLAIM, TASK_UNINTERRUPTIBLE, HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) clear_bit(DMZ_RECLAIM_TERMINATE, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) dmz_lock_metadata(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) * Select a cache or random write zone for reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) unsigned int idx, bool idle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) struct dm_zone *dzone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) struct dm_zone *zone, *maxw_z = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) struct list_head *zone_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) /* If we have cache zones select from the cache zone list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) if (zmd->nr_cache) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) zone_list = &zmd->map_cache_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) /* Try to relaim random zones, too, when idle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) if (idle && list_empty(zone_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) zone_list = &zmd->dev[idx].map_rnd_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) zone_list = &zmd->dev[idx].map_rnd_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) * Find the buffer zone with the heaviest weight or the first (oldest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) * data zone that can be reclaimed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) list_for_each_entry(zone, zone_list, link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) if (dmz_is_buf(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) dzone = zone->bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) if (!maxw_z || maxw_z->weight < dzone->weight)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) maxw_z = dzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) dzone = zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) if (dmz_lock_zone_reclaim(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) return dzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) if (maxw_z && dmz_lock_zone_reclaim(maxw_z))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) return maxw_z;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) * If we come here, none of the zones inspected could be locked for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) * reclaim. Try again, being more aggressive, that is, find the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) * first zone that can be reclaimed regardless of its weitght.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) list_for_each_entry(zone, zone_list, link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) if (dmz_is_buf(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) dzone = zone->bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) dzone = zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) if (dmz_lock_zone_reclaim(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) return dzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) * Select a buffered sequential zone for reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) unsigned int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) struct dm_zone *zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) if (!zone->bzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) if (dmz_lock_zone_reclaim(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) return zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) * Select a zone for reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) unsigned int dev_idx, bool idle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) struct dm_zone *zone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) * Search for a zone candidate to reclaim: 2 cases are possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) * (1) There is no free sequential zones. Then a random data zone
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) * cannot be reclaimed. So choose a sequential zone to reclaim so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) * that afterward a random zone can be reclaimed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) * (2) At least one free sequential zone is available, then choose
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) * the oldest random zone (data or buffer) that can be locked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) if (list_empty(&zmd->reserved_seq_zones_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) zone = dmz_get_seq_zone_for_reclaim(zmd, dev_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) if (!zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) zone = dmz_get_rnd_zone_for_reclaim(zmd, dev_idx, idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) return zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) * Get the zone mapping a chunk, if the chunk is mapped already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) * If no mapping exist and the operation is WRITE, a zone is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) * allocated and used to map the chunk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) * The zone returned will be set to the active state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk, int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) struct dmz_map *dmap = (struct dmz_map *) dmap_mblk->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) int dmap_idx = chunk & DMZ_MAP_ENTRIES_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) unsigned int dzone_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) struct dm_zone *dzone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) /* Get the chunk mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) dzone_id = le32_to_cpu(dmap[dmap_idx].dzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) if (dzone_id == DMZ_MAP_UNMAPPED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) * Read or discard in unmapped chunks are fine. But for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) * writes, we need a mapping, so get one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) if (op != REQ_OP_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) /* Allocate a random zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) dzone = dmz_alloc_zone(zmd, 0, alloc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) if (!dzone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) if (dmz_dev_is_dying(zmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) dzone = ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) dmz_wait_for_free_zones(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) dmz_map_zone(zmd, dzone, chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) /* The chunk is already mapped: get the mapping zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) dzone = dmz_get(zmd, dzone_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) if (!dzone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) dzone = ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) if (dzone->chunk != chunk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) dzone = ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) /* Repair write pointer if the sequential dzone has error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) if (dmz_seq_write_err(dzone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) ret = dmz_handle_seq_write_err(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) dzone = ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) clear_bit(DMZ_SEQ_WRITE_ERR, &dzone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) * If the zone is being reclaimed, the chunk mapping may change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) * to a different zone. So wait for reclaim and retry. Otherwise,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) * activate the zone (this will prevent reclaim from touching it).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) if (dmz_in_reclaim(dzone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) dmz_wait_for_reclaim(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) dmz_activate_zone(dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) dmz_lru_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) return dzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) * Write and discard change the block validity of data zones and their buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) * zones. Check here that valid blocks are still present. If all blocks are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) * invalid, the zones can be unmapped on the fly without waiting for reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) * to do it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) struct dm_zone *bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) bzone = dzone->bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) if (bzone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) if (dmz_weight(bzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) dmz_lru_zone(zmd, bzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) /* Empty buffer zone: reclaim it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) dmz_unmap_zone(zmd, bzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) dmz_free_zone(zmd, bzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) bzone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) /* Deactivate the data zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) dmz_deactivate_zone(dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) if (dmz_is_active(dzone) || bzone || dmz_weight(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) dmz_lru_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) /* Unbuffered inactive empty data zone: reclaim it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) dmz_unmap_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) dmz_free_zone(zmd, dzone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) * Allocate and map a random zone to buffer a chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) * already mapped to a sequential zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) struct dm_zone *dzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) struct dm_zone *bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) dmz_lock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) bzone = dzone->bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) if (bzone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) /* Allocate a random zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) bzone = dmz_alloc_zone(zmd, 0, alloc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) if (!bzone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) if (dmz_dev_is_dying(zmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) bzone = ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) dmz_wait_for_free_zones(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) /* Update the chunk mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) dmz_set_chunk_mapping(zmd, dzone->chunk, dzone->id, bzone->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) set_bit(DMZ_BUF, &bzone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) bzone->chunk = dzone->chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) bzone->bzone = dzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) dzone->bzone = bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) if (dmz_is_cache(bzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) list_add_tail(&bzone->link, &zmd->map_cache_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) list_add_tail(&bzone->link, &bzone->dev->map_rnd_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) dmz_unlock_map(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) return bzone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) * Get an unmapped (free) zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) * This must be called with the mapping lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) struct list_head *list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) struct dm_zone *zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) /* Schedule reclaim to ensure free zones are available */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) if (!(flags & DMZ_ALLOC_RECLAIM)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) for (i = 0; i < zmd->nr_devs; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) dmz_schedule_reclaim(zmd->dev[i].reclaim);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) if (flags & DMZ_ALLOC_CACHE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) list = &zmd->unmap_cache_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) else if (flags & DMZ_ALLOC_RND)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) list = &zmd->dev[dev_idx].unmap_rnd_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) list = &zmd->dev[dev_idx].unmap_seq_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) if (list_empty(list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) * No free zone: return NULL if this is for not reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) if (!(flags & DMZ_ALLOC_RECLAIM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) * Try to allocate from other devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) if (i < zmd->nr_devs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) dev_idx = (dev_idx + 1) % zmd->nr_devs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) * Fallback to the reserved sequential zones
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) struct dm_zone, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) if (zone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) list_del_init(&zone->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) atomic_dec(&zmd->nr_reserved_seq_zones);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) return zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) zone = list_first_entry(list, struct dm_zone, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) list_del_init(&zone->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) if (dmz_is_cache(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) atomic_dec(&zmd->unmap_nr_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) else if (dmz_is_rnd(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) atomic_dec(&zone->dev->unmap_nr_rnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) atomic_dec(&zone->dev->unmap_nr_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) if (dmz_is_offline(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) dmz_zmd_warn(zmd, "Zone %u is offline", zone->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) zone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) if (dmz_is_meta(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) zone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) return zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) * Free a zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) * This must be called with the mapping lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) /* If this is a sequential zone, reset it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) if (dmz_is_seq(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) dmz_reset_zone(zmd, zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) /* Return the zone to its type unmap list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) if (dmz_is_cache(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) list_add_tail(&zone->link, &zmd->unmap_cache_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) atomic_inc(&zmd->unmap_nr_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) } else if (dmz_is_rnd(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) list_add_tail(&zone->link, &zone->dev->unmap_rnd_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) atomic_inc(&zone->dev->unmap_nr_rnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) } else if (dmz_is_reserved(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) list_add_tail(&zone->link, &zmd->reserved_seq_zones_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) atomic_inc(&zmd->nr_reserved_seq_zones);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) list_add_tail(&zone->link, &zone->dev->unmap_seq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) atomic_inc(&zone->dev->unmap_nr_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) wake_up_all(&zmd->free_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) * Map a chunk to a zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) * This must be called with the mapping lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) unsigned int chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) /* Set the chunk mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) dmz_set_chunk_mapping(zmd, chunk, dzone->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) DMZ_MAP_UNMAPPED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) dzone->chunk = chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) if (dmz_is_cache(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) list_add_tail(&dzone->link, &zmd->map_cache_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) else if (dmz_is_rnd(dzone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) list_add_tail(&dzone->link, &dzone->dev->map_rnd_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) list_add_tail(&dzone->link, &dzone->dev->map_seq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) * Unmap a zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) * This must be called with the mapping lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) unsigned int chunk = zone->chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) unsigned int dzone_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) if (chunk == DMZ_MAP_UNMAPPED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) /* Already unmapped */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) if (test_and_clear_bit(DMZ_BUF, &zone->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) * Unmapping the chunk buffer zone: clear only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) * the chunk buffer mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) dzone_id = zone->bzone->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) zone->bzone->bzone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) zone->bzone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) * Unmapping the chunk data zone: the zone must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) * not be buffered.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) if (WARN_ON(zone->bzone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) zone->bzone->bzone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) zone->bzone = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) dzone_id = DMZ_MAP_UNMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) dmz_set_chunk_mapping(zmd, chunk, dzone_id, DMZ_MAP_UNMAPPED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) zone->chunk = DMZ_MAP_UNMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) list_del_init(&zone->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) * Set @nr_bits bits in @bitmap starting from @bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) * Return the number of bits changed from 0 to 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) static unsigned int dmz_set_bits(unsigned long *bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) unsigned int bit, unsigned int nr_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) unsigned long *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) unsigned int end = bit + nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) unsigned int n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) while (bit < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) if (((bit & (BITS_PER_LONG - 1)) == 0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) ((end - bit) >= BITS_PER_LONG)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) /* Try to set the whole word at once */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) addr = bitmap + BIT_WORD(bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) if (*addr == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) *addr = ULONG_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) n += BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) bit += BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) if (!test_and_set_bit(bit, bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) n++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) bit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) * Get the bitmap block storing the bit for chunk_block in zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) struct dm_zone *zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) sector_t chunk_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) sector_t bitmap_block = 1 + zmd->nr_map_blocks +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) (sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) (chunk_block >> DMZ_BLOCK_SHIFT_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) return dmz_get_mblock(zmd, bitmap_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) * Copy the valid blocks bitmap of from_zone to the bitmap of to_zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) struct dm_zone *to_zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) struct dmz_mblock *from_mblk, *to_mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) sector_t chunk_block = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) /* Get the zones bitmap blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) while (chunk_block < zmd->zone_nr_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) from_mblk = dmz_get_bitmap(zmd, from_zone, chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) if (IS_ERR(from_mblk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) return PTR_ERR(from_mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) to_mblk = dmz_get_bitmap(zmd, to_zone, chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) if (IS_ERR(to_mblk)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) dmz_release_mblock(zmd, from_mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) return PTR_ERR(to_mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) memcpy(to_mblk->data, from_mblk->data, DMZ_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) dmz_dirty_mblock(zmd, to_mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) dmz_release_mblock(zmd, to_mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) dmz_release_mblock(zmd, from_mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) chunk_block += zmd->zone_bits_per_mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) to_zone->weight = from_zone->weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) * Merge the valid blocks bitmap of from_zone into the bitmap of to_zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) * starting from chunk_block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) struct dm_zone *to_zone, sector_t chunk_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) unsigned int nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) /* Get the zones bitmap blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) while (chunk_block < zmd->zone_nr_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) /* Get a valid region from the source zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) ret = dmz_first_valid_block(zmd, from_zone, &chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) nr_blocks = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) ret = dmz_validate_blocks(zmd, to_zone, chunk_block, nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) chunk_block += nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) * Validate all the blocks in the range [block..block+nr_blocks-1].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) sector_t chunk_block, unsigned int nr_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) unsigned int count, bit, nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) unsigned int n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) dmz_zmd_debug(zmd, "=> VALIDATE zone %u, block %llu, %u blocks",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) zone->id, (unsigned long long)chunk_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) WARN_ON(chunk_block + nr_blocks > zone_nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) while (nr_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) /* Get bitmap block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) mblk = dmz_get_bitmap(zmd, zone, chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) if (IS_ERR(mblk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) return PTR_ERR(mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) /* Set bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) bit = chunk_block & DMZ_BLOCK_MASK_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) count = dmz_set_bits((unsigned long *)mblk->data, bit, nr_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) if (count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) dmz_dirty_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) n += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) dmz_release_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) nr_blocks -= nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) chunk_block += nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) if (likely(zone->weight + n <= zone_nr_blocks))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) zone->weight += n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) dmz_zmd_warn(zmd, "Zone %u: weight %u should be <= %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) zone->id, zone->weight,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) zone_nr_blocks - n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) zone->weight = zone_nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) * Clear nr_bits bits in bitmap starting from bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) * Return the number of bits cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) static int dmz_clear_bits(unsigned long *bitmap, int bit, int nr_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) unsigned long *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) int end = bit + nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) int n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) while (bit < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) if (((bit & (BITS_PER_LONG - 1)) == 0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) ((end - bit) >= BITS_PER_LONG)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) /* Try to clear whole word at once */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) addr = bitmap + BIT_WORD(bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) if (*addr == ULONG_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) *addr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) n += BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) bit += BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) if (test_and_clear_bit(bit, bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) n++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) bit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) * Invalidate all the blocks in the range [block..block+nr_blocks-1].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) sector_t chunk_block, unsigned int nr_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) unsigned int count, bit, nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) unsigned int n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) dmz_zmd_debug(zmd, "=> INVALIDATE zone %u, block %llu, %u blocks",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) zone->id, (u64)chunk_block, nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) while (nr_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) /* Get bitmap block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) mblk = dmz_get_bitmap(zmd, zone, chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) if (IS_ERR(mblk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) return PTR_ERR(mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) /* Clear bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) bit = chunk_block & DMZ_BLOCK_MASK_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) count = dmz_clear_bits((unsigned long *)mblk->data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) bit, nr_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) if (count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) dmz_dirty_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) n += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) dmz_release_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) nr_blocks -= nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) chunk_block += nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) if (zone->weight >= n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) zone->weight -= n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) dmz_zmd_warn(zmd, "Zone %u: weight %u should be >= %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) zone->id, zone->weight, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) zone->weight = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) * Get a block bit value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) sector_t chunk_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) WARN_ON(chunk_block >= zmd->zone_nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) /* Get bitmap block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) mblk = dmz_get_bitmap(zmd, zone, chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) if (IS_ERR(mblk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) return PTR_ERR(mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) /* Get offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) ret = test_bit(chunk_block & DMZ_BLOCK_MASK_BITS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) (unsigned long *) mblk->data) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) dmz_release_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) * Return the number of blocks from chunk_block to the first block with a bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) * value specified by set. Search at most nr_blocks blocks from chunk_block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) sector_t chunk_block, unsigned int nr_blocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) unsigned int bit, set_bit, nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) unsigned int zone_bits = zmd->zone_bits_per_mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) unsigned long *bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) int n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) while (nr_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) /* Get bitmap block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) mblk = dmz_get_bitmap(zmd, zone, chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) if (IS_ERR(mblk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) return PTR_ERR(mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) /* Get offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) bitmap = (unsigned long *) mblk->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) bit = chunk_block & DMZ_BLOCK_MASK_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) nr_bits = min(nr_blocks, zone_bits - bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) if (set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) set_bit = find_next_bit(bitmap, zone_bits, bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) set_bit = find_next_zero_bit(bitmap, zone_bits, bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) dmz_release_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) n += set_bit - bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) if (set_bit < zone_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) nr_blocks -= nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) chunk_block += nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) * Test if chunk_block is valid. If it is, the number of consecutive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) * valid blocks from chunk_block will be returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) sector_t chunk_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) int valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) valid = dmz_test_block(zmd, zone, chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) if (valid <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) return valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) /* The block is valid: get the number of valid blocks from block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) return dmz_to_next_set_block(zmd, zone, chunk_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) zmd->zone_nr_blocks - chunk_block, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) * Find the first valid block from @chunk_block in @zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) * If such a block is found, its number is returned using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) * @chunk_block and the total number of valid blocks from @chunk_block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) * is returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) sector_t *chunk_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) sector_t start_block = *chunk_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) ret = dmz_to_next_set_block(zmd, zone, start_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) zmd->zone_nr_blocks - start_block, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) start_block += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) *chunk_block = start_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) return dmz_to_next_set_block(zmd, zone, start_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) zmd->zone_nr_blocks - start_block, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) * Count the number of bits set starting from bit up to bit + nr_bits - 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) static int dmz_count_bits(void *bitmap, int bit, int nr_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) unsigned long *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) int end = bit + nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) int n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) while (bit < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) if (((bit & (BITS_PER_LONG - 1)) == 0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) ((end - bit) >= BITS_PER_LONG)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) addr = (unsigned long *)bitmap + BIT_WORD(bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) if (*addr == ULONG_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) n += BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) bit += BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) if (test_bit(bit, bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) n++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) bit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) * Get a zone weight.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) struct dmz_mblock *mblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) sector_t chunk_block = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) unsigned int bit, nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) unsigned int nr_blocks = zmd->zone_nr_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) void *bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) int n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) while (nr_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) /* Get bitmap block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) mblk = dmz_get_bitmap(zmd, zone, chunk_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) if (IS_ERR(mblk)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) /* Count bits in this block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) bitmap = mblk->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) bit = chunk_block & DMZ_BLOCK_MASK_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) n += dmz_count_bits(bitmap, bit, nr_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) dmz_release_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) nr_blocks -= nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) chunk_block += nr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) zone->weight = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) * Cleanup the zoned metadata resources.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) static void dmz_cleanup_metadata(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) struct rb_root *root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) struct dmz_mblock *mblk, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) /* Release zone mapping resources */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) if (zmd->map_mblk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) for (i = 0; i < zmd->nr_map_blocks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) dmz_release_mblock(zmd, zmd->map_mblk[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) kfree(zmd->map_mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) zmd->map_mblk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) /* Release super blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) for (i = 0; i < 2; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) if (zmd->sb[i].mblk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) dmz_free_mblock(zmd, zmd->sb[i].mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) zmd->sb[i].mblk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) /* Free cached blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) while (!list_empty(&zmd->mblk_dirty_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) mblk = list_first_entry(&zmd->mblk_dirty_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) struct dmz_mblock, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) dmz_zmd_warn(zmd, "mblock %llu still in dirty list (ref %u)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) (u64)mblk->no, mblk->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) list_del_init(&mblk->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) rb_erase(&mblk->node, &zmd->mblk_rbtree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) dmz_free_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) while (!list_empty(&zmd->mblk_lru_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) mblk = list_first_entry(&zmd->mblk_lru_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) struct dmz_mblock, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) list_del_init(&mblk->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) rb_erase(&mblk->node, &zmd->mblk_rbtree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) dmz_free_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) /* Sanity checks: the mblock rbtree should now be empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) root = &zmd->mblk_rbtree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) rbtree_postorder_for_each_entry_safe(mblk, next, root, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) dmz_zmd_warn(zmd, "mblock %llu ref %u still in rbtree",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) (u64)mblk->no, mblk->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) mblk->ref = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) dmz_free_mblock(zmd, mblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) /* Free the zone descriptors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) dmz_drop_zones(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) mutex_destroy(&zmd->mblk_flush_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) mutex_destroy(&zmd->map_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) static void dmz_print_dev(struct dmz_metadata *zmd, int num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) struct dmz_dev *dev = &zmd->dev[num];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) if (bdev_zoned_model(dev->bdev) == BLK_ZONED_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) dmz_dev_info(dev, "Regular block device");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) dmz_dev_info(dev, "Host-%s zoned block device",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) bdev_zoned_model(dev->bdev) == BLK_ZONED_HA ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) "aware" : "managed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) if (zmd->sb_version > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) sector_t sector_offset =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) dev->zone_offset << zmd->zone_nr_sectors_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) dmz_dev_info(dev, " %llu 512-byte logical sectors (offset %llu)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) (u64)dev->capacity, (u64)sector_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors (offset %llu)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) dev->nr_zones, (u64)zmd->zone_nr_sectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) (u64)dev->zone_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) dmz_dev_info(dev, " %llu 512-byte logical sectors",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) (u64)dev->capacity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) dev->nr_zones, (u64)zmd->zone_nr_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) * Initialize the zoned metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) int dmz_ctr_metadata(struct dmz_dev *dev, int num_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) struct dmz_metadata **metadata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) const char *devname)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) struct dmz_metadata *zmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) struct dm_zone *zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) zmd = kzalloc(sizeof(struct dmz_metadata), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) if (!zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) strcpy(zmd->devname, devname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) zmd->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) zmd->nr_devs = num_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) zmd->mblk_rbtree = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) init_rwsem(&zmd->mblk_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) mutex_init(&zmd->mblk_flush_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) spin_lock_init(&zmd->mblk_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) INIT_LIST_HEAD(&zmd->mblk_lru_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) INIT_LIST_HEAD(&zmd->mblk_dirty_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) mutex_init(&zmd->map_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) atomic_set(&zmd->unmap_nr_cache, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) INIT_LIST_HEAD(&zmd->unmap_cache_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) INIT_LIST_HEAD(&zmd->map_cache_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) atomic_set(&zmd->nr_reserved_seq_zones, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) INIT_LIST_HEAD(&zmd->reserved_seq_zones_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) init_waitqueue_head(&zmd->free_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) /* Initialize zone descriptors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) ret = dmz_init_zones(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) /* Get super block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) ret = dmz_load_sb(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) /* Set metadata zones starting from sb_zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) for (i = 0; i < zmd->nr_meta_zones << 1; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) zone = dmz_get(zmd, zmd->sb[0].zone->id + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) if (!zone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) dmz_zmd_err(zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) "metadata zone %u not present", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) ret = -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) if (!dmz_is_rnd(zone) && !dmz_is_cache(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) dmz_zmd_err(zmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) "metadata zone %d is not random", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) ret = -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) set_bit(DMZ_META, &zone->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) /* Load mapping table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) ret = dmz_load_mapping(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) * Cache size boundaries: allow at least 2 super blocks, the chunk map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) * blocks and enough blocks to be able to cache the bitmap blocks of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) * up to 16 zones when idle (min_nr_mblks). Otherwise, if busy, allow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) * the cache to add 512 more metadata blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) zmd->max_nr_mblks = zmd->min_nr_mblks + 512;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) zmd->mblk_shrinker.count_objects = dmz_mblock_shrinker_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) zmd->mblk_shrinker.scan_objects = dmz_mblock_shrinker_scan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) zmd->mblk_shrinker.seeks = DEFAULT_SEEKS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) /* Metadata cache shrinker */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) ret = register_shrinker(&zmd->mblk_shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) dmz_zmd_err(zmd, "Register metadata cache shrinker failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) for (i = 0; i < zmd->nr_devs; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) dmz_print_dev(zmd, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) dmz_zmd_info(zmd, " %u zones of %llu 512-byte logical sectors",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) zmd->nr_zones, (u64)zmd->zone_nr_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) dmz_zmd_debug(zmd, " %u metadata zones",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) zmd->nr_meta_zones * 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) dmz_zmd_debug(zmd, " %u data zones for %u chunks",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) zmd->nr_data_zones, zmd->nr_chunks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) dmz_zmd_debug(zmd, " %u cache zones (%u unmapped)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) for (i = 0; i < zmd->nr_devs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) dmz_zmd_debug(zmd, " %u random zones (%u unmapped)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) dmz_nr_rnd_zones(zmd, i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) dmz_nr_unmap_rnd_zones(zmd, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) dmz_zmd_debug(zmd, " %u sequential zones (%u unmapped)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) dmz_nr_seq_zones(zmd, i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) dmz_nr_unmap_seq_zones(zmd, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) dmz_zmd_debug(zmd, " %u reserved sequential data zones",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) zmd->nr_reserved_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) dmz_zmd_debug(zmd, "Format:");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) zmd->nr_meta_blocks, zmd->max_nr_mblks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) dmz_zmd_debug(zmd, " %u data zone mapping blocks",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) zmd->nr_map_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) dmz_zmd_debug(zmd, " %u bitmap blocks",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) zmd->nr_bitmap_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) *metadata = zmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) dmz_cleanup_metadata(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) kfree(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) *metadata = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) * Cleanup the zoned metadata resources.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) void dmz_dtr_metadata(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) unregister_shrinker(&zmd->mblk_shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) dmz_cleanup_metadata(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) kfree(zmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) * Check zone information on resume.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) int dmz_resume_metadata(struct dmz_metadata *zmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) struct dm_zone *zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) sector_t wp_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) /* Check zones */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) for (i = 0; i < zmd->nr_zones; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) zone = dmz_get(zmd, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) if (!zone) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) dmz_zmd_err(zmd, "Unable to get zone %u", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) wp_block = zone->wp_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) ret = dmz_update_zone(zmd, zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) dmz_zmd_err(zmd, "Broken zone %u", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) if (dmz_is_offline(zone)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) dmz_zmd_warn(zmd, "Zone %u is offline", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) /* Check write pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) if (!dmz_is_seq(zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) zone->wp_block = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) else if (zone->wp_block != wp_block) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) dmz_zmd_err(zmd, "Zone %u: Invalid wp (%llu / %llu)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) i, (u64)zone->wp_block, (u64)wp_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) zone->wp_block = wp_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) dmz_invalidate_blocks(zmd, zone, zone->wp_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) zmd->zone_nr_blocks - zone->wp_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags