^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Block Translation Table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (c) 2014-2015, Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/hdreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/genhd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/sizes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/ndctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/nd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/backing-dev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "btt.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "nd.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) enum log_ent_request {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) LOG_NEW_ENT = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) LOG_OLD_ENT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static struct device *to_dev(struct arena_info *arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) return &arena->nd_btt->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) static u64 adjust_initial_offset(struct nd_btt *nd_btt, u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) return offset + nd_btt->initial_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) static int arena_read_bytes(struct arena_info *arena, resource_size_t offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) void *buf, size_t n, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct nd_btt *nd_btt = arena->nd_btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) struct nd_namespace_common *ndns = nd_btt->ndns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) /* arena offsets may be shifted from the base of the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) offset = adjust_initial_offset(nd_btt, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) return nvdimm_read_bytes(ndns, offset, buf, n, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static int arena_write_bytes(struct arena_info *arena, resource_size_t offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) void *buf, size_t n, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct nd_btt *nd_btt = arena->nd_btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct nd_namespace_common *ndns = nd_btt->ndns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /* arena offsets may be shifted from the base of the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) offset = adjust_initial_offset(nd_btt, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return nvdimm_write_bytes(ndns, offset, buf, n, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static int btt_info_write(struct arena_info *arena, struct btt_sb *super)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * infooff and info2off should always be at least 512B aligned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * We rely on that to make sure rw_bytes does error clearing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * correctly, so make sure that is the case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) dev_WARN_ONCE(to_dev(arena), !IS_ALIGNED(arena->infooff, 512),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) "arena->infooff: %#llx is unaligned\n", arena->infooff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) dev_WARN_ONCE(to_dev(arena), !IS_ALIGNED(arena->info2off, 512),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) "arena->info2off: %#llx is unaligned\n", arena->info2off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) ret = arena_write_bytes(arena, arena->info2off, super,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) sizeof(struct btt_sb), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) return arena_write_bytes(arena, arena->infooff, super,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) sizeof(struct btt_sb), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static int btt_info_read(struct arena_info *arena, struct btt_sb *super)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) return arena_read_bytes(arena, arena->infooff, super,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) sizeof(struct btt_sb), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * 'raw' version of btt_map write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * Assumptions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * mapping is in little-endian
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * mapping contains 'E' and 'Z' flags as desired
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static int __btt_map_write(struct arena_info *arena, u32 lba, __le32 mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) u64 ns_off = arena->mapoff + (lba * MAP_ENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) if (unlikely(lba >= arena->external_nlba))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) dev_err_ratelimited(to_dev(arena),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) "%s: lba %#x out of range (max: %#x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) __func__, lba, arena->external_nlba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) return arena_write_bytes(arena, ns_off, &mapping, MAP_ENT_SIZE, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static int btt_map_write(struct arena_info *arena, u32 lba, u32 mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) u32 z_flag, u32 e_flag, unsigned long rwb_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) u32 ze;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) __le32 mapping_le;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * This 'mapping' is supposed to be just the LBA mapping, without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * any flags set, so strip the flag bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) mapping = ent_lba(mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ze = (z_flag << 1) + e_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) switch (ze) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * We want to set neither of the Z or E flags, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * in the actual layout, this means setting the bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * positions of both to '1' to indicate a 'normal'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * map entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) mapping |= MAP_ENT_NORMAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) case 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) mapping |= (1 << MAP_ERR_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) case 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) mapping |= (1 << MAP_TRIM_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * The case where Z and E are both sent in as '1' could be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * construed as a valid 'normal' case, but we decide not to,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * to avoid confusion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) dev_err_ratelimited(to_dev(arena),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) "Invalid use of Z and E flags\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) mapping_le = cpu_to_le32(mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return __btt_map_write(arena, lba, mapping_le, rwb_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static int btt_map_read(struct arena_info *arena, u32 lba, u32 *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) int *trim, int *error, unsigned long rwb_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) __le32 in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) u32 raw_mapping, postmap, ze, z_flag, e_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) u64 ns_off = arena->mapoff + (lba * MAP_ENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) if (unlikely(lba >= arena->external_nlba))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) dev_err_ratelimited(to_dev(arena),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) "%s: lba %#x out of range (max: %#x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) __func__, lba, arena->external_nlba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) ret = arena_read_bytes(arena, ns_off, &in, MAP_ENT_SIZE, rwb_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) raw_mapping = le32_to_cpu(in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) z_flag = ent_z_flag(raw_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) e_flag = ent_e_flag(raw_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) ze = (z_flag << 1) + e_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) postmap = ent_lba(raw_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /* Reuse the {z,e}_flag variables for *trim and *error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) z_flag = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) e_flag = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) switch (ze) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) /* Initial state. Return postmap = premap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) *mapping = lba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) case 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) *mapping = postmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) e_flag = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) case 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) *mapping = postmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) z_flag = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) case 3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) *mapping = postmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (trim)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) *trim = z_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) *error = e_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) static int btt_log_group_read(struct arena_info *arena, u32 lane,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) struct log_group *log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return arena_read_bytes(arena,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) arena->logoff + (lane * LOG_GRP_SIZE), log,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) LOG_GRP_SIZE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) static struct dentry *debugfs_root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static void arena_debugfs_init(struct arena_info *a, struct dentry *parent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) char dirname[32];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) struct dentry *d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /* If for some reason, parent bttN was not created, exit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (!parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) snprintf(dirname, 32, "arena%d", idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) d = debugfs_create_dir(dirname, parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (IS_ERR_OR_NULL(d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) a->debugfs_dir = d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) debugfs_create_x64("size", S_IRUGO, d, &a->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) debugfs_create_x64("external_lba_start", S_IRUGO, d,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) &a->external_lba_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) debugfs_create_x32("internal_nlba", S_IRUGO, d, &a->internal_nlba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) debugfs_create_u32("internal_lbasize", S_IRUGO, d,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) &a->internal_lbasize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) debugfs_create_x32("external_nlba", S_IRUGO, d, &a->external_nlba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) debugfs_create_u32("external_lbasize", S_IRUGO, d,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) &a->external_lbasize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) debugfs_create_u32("nfree", S_IRUGO, d, &a->nfree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) debugfs_create_u16("version_major", S_IRUGO, d, &a->version_major);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) debugfs_create_u16("version_minor", S_IRUGO, d, &a->version_minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) debugfs_create_x64("nextoff", S_IRUGO, d, &a->nextoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) debugfs_create_x64("infooff", S_IRUGO, d, &a->infooff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) debugfs_create_x64("dataoff", S_IRUGO, d, &a->dataoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) debugfs_create_x64("mapoff", S_IRUGO, d, &a->mapoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) debugfs_create_x64("logoff", S_IRUGO, d, &a->logoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) debugfs_create_x64("info2off", S_IRUGO, d, &a->info2off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) debugfs_create_x32("flags", S_IRUGO, d, &a->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) debugfs_create_u32("log_index_0", S_IRUGO, d, &a->log_index[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) debugfs_create_u32("log_index_1", S_IRUGO, d, &a->log_index[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) static void btt_debugfs_init(struct btt *btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct arena_info *arena;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) btt->debugfs_dir = debugfs_create_dir(dev_name(&btt->nd_btt->dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) debugfs_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (IS_ERR_OR_NULL(btt->debugfs_dir))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) list_for_each_entry(arena, &btt->arena_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) arena_debugfs_init(arena, btt->debugfs_dir, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static u32 log_seq(struct log_group *log, int log_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return le32_to_cpu(log->ent[log_idx].seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * This function accepts two log entries, and uses the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * sequence number to find the 'older' entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * It also updates the sequence number in this old entry to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * make it the 'new' one if the mark_flag is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * Finally, it returns which of the entries was the older one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * TODO The logic feels a bit kludge-y. make it better..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) static int btt_log_get_old(struct arena_info *a, struct log_group *log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) int idx0 = a->log_index[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) int idx1 = a->log_index[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) int old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * the first ever time this is seen, the entry goes into [0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * the next time, the following logic works out to put this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * (next) entry into [1]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (log_seq(log, idx0) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) log->ent[idx0].seq = cpu_to_le32(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (log_seq(log, idx0) == log_seq(log, idx1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (log_seq(log, idx0) + log_seq(log, idx1) > 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (log_seq(log, idx0) < log_seq(log, idx1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if ((log_seq(log, idx1) - log_seq(log, idx0)) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) old = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) old = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if ((log_seq(log, idx0) - log_seq(log, idx1)) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) old = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) old = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * This function copies the desired (old/new) log entry into ent if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * it is not NULL. It returns the sub-slot number (0 or 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * where the desired log entry was found. Negative return values
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * indicate errors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) static int btt_log_read(struct arena_info *arena, u32 lane,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) struct log_entry *ent, int old_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) int old_ent, ret_ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) struct log_group log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) ret = btt_log_group_read(arena, lane, &log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) old_ent = btt_log_get_old(arena, &log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (old_ent < 0 || old_ent > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) dev_err(to_dev(arena),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) "log corruption (%d): lane %d seq [%d, %d]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) old_ent, lane, log.ent[arena->log_index[0]].seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) log.ent[arena->log_index[1]].seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) /* TODO set error state? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) ret_ent = (old_flag ? old_ent : (1 - old_ent));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (ent != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) memcpy(ent, &log.ent[arena->log_index[ret_ent]], LOG_ENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return ret_ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * This function commits a log entry to media
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * It does _not_ prepare the freelist entry for the next write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * btt_flog_write is the wrapper for updating the freelist elements
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) static int __btt_log_write(struct arena_info *arena, u32 lane,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) u32 sub, struct log_entry *ent, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) u32 group_slot = arena->log_index[sub];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) unsigned int log_half = LOG_ENT_SIZE / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) void *src = ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) u64 ns_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) ns_off = arena->logoff + (lane * LOG_GRP_SIZE) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) (group_slot * LOG_ENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /* split the 16B write into atomic, durable halves */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) ret = arena_write_bytes(arena, ns_off, src, log_half, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) ns_off += log_half;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) src += log_half;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return arena_write_bytes(arena, ns_off, src, log_half, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) static int btt_flog_write(struct arena_info *arena, u32 lane, u32 sub,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct log_entry *ent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) ret = __btt_log_write(arena, lane, sub, ent, NVDIMM_IO_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) /* prepare the next free entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) arena->freelist[lane].sub = 1 - arena->freelist[lane].sub;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (++(arena->freelist[lane].seq) == 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) arena->freelist[lane].seq = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) if (ent_e_flag(le32_to_cpu(ent->old_map)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) arena->freelist[lane].has_err = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) arena->freelist[lane].block = ent_lba(le32_to_cpu(ent->old_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * This function initializes the BTT map to the initial state, which is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * all-zeroes, and indicates an identity mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static int btt_map_init(struct arena_info *arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) void *zerobuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) size_t offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) size_t chunk_size = SZ_2M;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) size_t mapsize = arena->logoff - arena->mapoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) zerobuf = kzalloc(chunk_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (!zerobuf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * mapoff should always be at least 512B aligned. We rely on that to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * make sure rw_bytes does error clearing correctly, so make sure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * is the case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) dev_WARN_ONCE(to_dev(arena), !IS_ALIGNED(arena->mapoff, 512),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) "arena->mapoff: %#llx is unaligned\n", arena->mapoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) while (mapsize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) size_t size = min(mapsize, chunk_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) dev_WARN_ONCE(to_dev(arena), size < 512,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) "chunk size: %#zx is unaligned\n", size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) ret = arena_write_bytes(arena, arena->mapoff + offset, zerobuf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) size, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) offset += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) mapsize -= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) kfree(zerobuf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * This function initializes the BTT log with 'fake' entries pointing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * to the initial reserved set of blocks as being free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) static int btt_log_init(struct arena_info *arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) size_t logsize = arena->info2off - arena->logoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) size_t chunk_size = SZ_4K, offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) struct log_entry ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) void *zerobuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) zerobuf = kzalloc(chunk_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (!zerobuf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * logoff should always be at least 512B aligned. We rely on that to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * make sure rw_bytes does error clearing correctly, so make sure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * is the case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) dev_WARN_ONCE(to_dev(arena), !IS_ALIGNED(arena->logoff, 512),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) "arena->logoff: %#llx is unaligned\n", arena->logoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) while (logsize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) size_t size = min(logsize, chunk_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) dev_WARN_ONCE(to_dev(arena), size < 512,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) "chunk size: %#zx is unaligned\n", size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) ret = arena_write_bytes(arena, arena->logoff + offset, zerobuf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) size, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) offset += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) logsize -= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) for (i = 0; i < arena->nfree; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) ent.lba = cpu_to_le32(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) ent.old_map = cpu_to_le32(arena->external_nlba + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) ent.new_map = cpu_to_le32(arena->external_nlba + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) ent.seq = cpu_to_le32(LOG_SEQ_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) ret = __btt_log_write(arena, i, 0, &ent, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) kfree(zerobuf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) static u64 to_namespace_offset(struct arena_info *arena, u64 lba)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) return arena->dataoff + ((u64)lba * arena->internal_lbasize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) static int arena_clear_freelist_error(struct arena_info *arena, u32 lane)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (arena->freelist[lane].has_err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) void *zero_page = page_address(ZERO_PAGE(0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) u32 lba = arena->freelist[lane].block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) u64 nsoff = to_namespace_offset(arena, lba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) unsigned long len = arena->sector_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) mutex_lock(&arena->err_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) while (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) unsigned long chunk = min(len, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) ret = arena_write_bytes(arena, nsoff, zero_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) chunk, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) len -= chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) nsoff += chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (len == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) arena->freelist[lane].has_err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) mutex_unlock(&arena->err_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) static int btt_freelist_init(struct arena_info *arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) int new, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) struct log_entry log_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) u32 i, map_entry, log_oldmap, log_newmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) arena->freelist = kcalloc(arena->nfree, sizeof(struct free_entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (!arena->freelist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) for (i = 0; i < arena->nfree; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) new = btt_log_read(arena, i, &log_new, LOG_NEW_ENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (new < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) /* old and new map entries with any flags stripped out */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) log_oldmap = ent_lba(le32_to_cpu(log_new.old_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) log_newmap = ent_lba(le32_to_cpu(log_new.new_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) /* sub points to the next one to be overwritten */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) arena->freelist[i].sub = 1 - new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) arena->freelist[i].seq = nd_inc_seq(le32_to_cpu(log_new.seq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) arena->freelist[i].block = log_oldmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * FIXME: if error clearing fails during init, we want to make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * the BTT read-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) if (ent_e_flag(le32_to_cpu(log_new.old_map)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) !ent_normal(le32_to_cpu(log_new.old_map))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) arena->freelist[i].has_err = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) ret = arena_clear_freelist_error(arena, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) dev_err_ratelimited(to_dev(arena),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) "Unable to clear known errors\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /* This implies a newly created or untouched flog entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (log_oldmap == log_newmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) /* Check if map recovery is needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) ret = btt_map_read(arena, le32_to_cpu(log_new.lba), &map_entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) NULL, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * The map_entry from btt_read_map is stripped of any flag bits,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) * so use the stripped out versions from the log as well for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) * testing whether recovery is needed. For restoration, use the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) * 'raw' version of the log entries as that captured what we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * were going to write originally.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if ((log_newmap != map_entry) && (log_oldmap == map_entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * Last transaction wrote the flog, but wasn't able
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * to complete the map write. So fix up the map.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) ret = btt_map_write(arena, le32_to_cpu(log_new.lba),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) le32_to_cpu(log_new.new_map), 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) static bool ent_is_padding(struct log_entry *ent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) return (ent->lba == 0) && (ent->old_map == 0) && (ent->new_map == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) && (ent->seq == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) * Detecting valid log indices: We read a log group (see the comments in btt.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) * for a description of a 'log_group' and its 'slots'), and iterate over its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * four slots. We expect that a padding slot will be all-zeroes, and use this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) * to detect a padding slot vs. an actual entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * If a log_group is in the initial state, i.e. hasn't been used since the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * creation of this BTT layout, it will have three of the four slots with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) * zeroes. We skip over these log_groups for the detection of log_index. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * all log_groups are in the initial state (i.e. the BTT has never been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * written to), it is safe to assume the 'new format' of log entries in slots
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * (0, 1).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) static int log_set_indices(struct arena_info *arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) bool idx_set = false, initial_state = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) int ret, log_index[2] = {-1, -1};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) u32 i, j, next_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct log_group log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) u32 pad_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) for (i = 0; i < arena->nfree; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) ret = btt_log_group_read(arena, i, &log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) for (j = 0; j < 4; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) if (!idx_set) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) if (ent_is_padding(&log.ent[j])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) pad_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) /* Skip if index has been recorded */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) if ((next_idx == 1) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) (j == log_index[0]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) /* valid entry, record index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) log_index[next_idx] = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) next_idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) if (next_idx == 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) /* two valid entries found */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) idx_set = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) } else if (next_idx > 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) /* too many valid indices */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) * once the indices have been set, just verify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * that all subsequent log groups are either in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * their initial state or follow the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * indices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) if (j == log_index[0]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) /* entry must be 'valid' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (ent_is_padding(&log.ent[j]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) } else if (j == log_index[1]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * log_index[1] can be padding if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * lane never got used and it is still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * in the initial state (three 'padding'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) /* entry must be invalid (padding) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (!ent_is_padding(&log.ent[j]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) * If any of the log_groups have more than one valid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) * non-padding entry, then the we are no longer in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) * initial_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) if (pad_count < 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) initial_state = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) pad_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) if (!initial_state && !idx_set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * If all the entries in the log were in the initial state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * assume new padding scheme
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (initial_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) log_index[1] = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * Only allow the known permutations of log/padding indices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * i.e. (0, 1), and (0, 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if ((log_index[0] == 0) && ((log_index[1] == 1) || (log_index[1] == 2)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) ; /* known index possibilities */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) dev_err(to_dev(arena), "Found an unknown padding scheme\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) arena->log_index[0] = log_index[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) arena->log_index[1] = log_index[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) dev_dbg(to_dev(arena), "log_index_0 = %d\n", log_index[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) dev_dbg(to_dev(arena), "log_index_1 = %d\n", log_index[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) static int btt_rtt_init(struct arena_info *arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) arena->rtt = kcalloc(arena->nfree, sizeof(u32), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (arena->rtt == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) static int btt_maplocks_init(struct arena_info *arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) arena->map_locks = kcalloc(arena->nfree, sizeof(struct aligned_lock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (!arena->map_locks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) for (i = 0; i < arena->nfree; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) spin_lock_init(&arena->map_locks[i].lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) static struct arena_info *alloc_arena(struct btt *btt, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) size_t start, size_t arena_off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) struct arena_info *arena;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) u64 logsize, mapsize, datasize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) u64 available = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) arena = kzalloc(sizeof(struct arena_info), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) if (!arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) arena->nd_btt = btt->nd_btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) arena->sector_size = btt->sector_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) mutex_init(&arena->err_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) if (!size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) return arena;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) arena->size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) arena->external_lba_start = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) arena->external_lbasize = btt->lbasize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) arena->internal_lbasize = roundup(arena->external_lbasize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) INT_LBASIZE_ALIGNMENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) arena->nfree = BTT_DEFAULT_NFREE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) arena->version_major = btt->nd_btt->version_major;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) arena->version_minor = btt->nd_btt->version_minor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (available % BTT_PG_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) available -= (available % BTT_PG_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) /* Two pages are reserved for the super block and its copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) available -= 2 * BTT_PG_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) /* The log takes a fixed amount of space based on nfree */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) logsize = roundup(arena->nfree * LOG_GRP_SIZE, BTT_PG_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) available -= logsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) /* Calculate optimal split between map and data area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) arena->internal_nlba = div_u64(available - BTT_PG_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) arena->internal_lbasize + MAP_ENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) arena->external_nlba = arena->internal_nlba - arena->nfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) mapsize = roundup((arena->external_nlba * MAP_ENT_SIZE), BTT_PG_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) datasize = available - mapsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) /* 'Absolute' values, relative to start of storage space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) arena->infooff = arena_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) arena->dataoff = arena->infooff + BTT_PG_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) arena->mapoff = arena->dataoff + datasize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) arena->logoff = arena->mapoff + mapsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) arena->info2off = arena->logoff + logsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) /* Default log indices are (0,1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) arena->log_index[0] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) arena->log_index[1] = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) return arena;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) static void free_arenas(struct btt *btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) struct arena_info *arena, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) list_for_each_entry_safe(arena, next, &btt->arena_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) list_del(&arena->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) kfree(arena->rtt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) kfree(arena->map_locks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) kfree(arena->freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) debugfs_remove_recursive(arena->debugfs_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) kfree(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * This function reads an existing valid btt superblock and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) * populates the corresponding arena_info struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) static void parse_arena_meta(struct arena_info *arena, struct btt_sb *super,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) u64 arena_off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) arena->internal_nlba = le32_to_cpu(super->internal_nlba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) arena->internal_lbasize = le32_to_cpu(super->internal_lbasize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) arena->external_nlba = le32_to_cpu(super->external_nlba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) arena->external_lbasize = le32_to_cpu(super->external_lbasize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) arena->nfree = le32_to_cpu(super->nfree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) arena->version_major = le16_to_cpu(super->version_major);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) arena->version_minor = le16_to_cpu(super->version_minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) arena->nextoff = (super->nextoff == 0) ? 0 : (arena_off +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) le64_to_cpu(super->nextoff));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) arena->infooff = arena_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) arena->dataoff = arena_off + le64_to_cpu(super->dataoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) arena->mapoff = arena_off + le64_to_cpu(super->mapoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) arena->logoff = arena_off + le64_to_cpu(super->logoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) arena->info2off = arena_off + le64_to_cpu(super->info2off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) arena->size = (le64_to_cpu(super->nextoff) > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) ? (le64_to_cpu(super->nextoff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) : (arena->info2off - arena->infooff + BTT_PG_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) arena->flags = le32_to_cpu(super->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) static int discover_arenas(struct btt *btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) struct arena_info *arena;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) struct btt_sb *super;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) size_t remaining = btt->rawsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) u64 cur_nlba = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) size_t cur_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) int num_arenas = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) super = kzalloc(sizeof(*super), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) if (!super)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) while (remaining) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) /* Alloc memory for arena */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) arena = alloc_arena(btt, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) if (!arena) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) goto out_super;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) arena->infooff = cur_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) ret = btt_info_read(arena, super);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) if (!nd_btt_arena_is_valid(btt->nd_btt, super)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) if (remaining == btt->rawsize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) btt->init_state = INIT_NOTFOUND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) dev_info(to_dev(arena), "No existing arenas\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) dev_err(to_dev(arena),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) "Found corrupted metadata!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) arena->external_lba_start = cur_nlba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) parse_arena_meta(arena, super, cur_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) ret = log_set_indices(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) dev_err(to_dev(arena),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) "Unable to deduce log/padding indices\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) ret = btt_freelist_init(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) ret = btt_rtt_init(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) ret = btt_maplocks_init(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) list_add_tail(&arena->list, &btt->arena_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) remaining -= arena->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) cur_off += arena->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) cur_nlba += arena->external_nlba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) num_arenas++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) if (arena->nextoff == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) btt->num_arenas = num_arenas;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) btt->nlba = cur_nlba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) btt->init_state = INIT_READY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) kfree(super);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) kfree(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) free_arenas(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) out_super:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) kfree(super);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) static int create_arenas(struct btt *btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) size_t remaining = btt->rawsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) size_t cur_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) while (remaining) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) struct arena_info *arena;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) size_t arena_size = min_t(u64, ARENA_MAX_SIZE, remaining);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) remaining -= arena_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) if (arena_size < ARENA_MIN_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) arena = alloc_arena(btt, arena_size, btt->nlba, cur_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) if (!arena) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) free_arenas(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) btt->nlba += arena->external_nlba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) if (remaining >= ARENA_MIN_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) arena->nextoff = arena->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) arena->nextoff = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) cur_off += arena_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) list_add_tail(&arena->list, &btt->arena_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) * This function completes arena initialization by writing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) * all the metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) * It is only called for an uninitialized arena when a write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) * to that arena occurs for the first time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) static int btt_arena_write_layout(struct arena_info *arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) u64 sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) struct btt_sb *super;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) struct nd_btt *nd_btt = arena->nd_btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) const u8 *parent_uuid = nd_dev_to_uuid(&nd_btt->ndns->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) ret = btt_map_init(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) ret = btt_log_init(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) super = kzalloc(sizeof(struct btt_sb), GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) if (!super)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) strncpy(super->signature, BTT_SIG, BTT_SIG_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) memcpy(super->uuid, nd_btt->uuid, 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) memcpy(super->parent_uuid, parent_uuid, 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) super->flags = cpu_to_le32(arena->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) super->version_major = cpu_to_le16(arena->version_major);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) super->version_minor = cpu_to_le16(arena->version_minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) super->external_lbasize = cpu_to_le32(arena->external_lbasize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) super->external_nlba = cpu_to_le32(arena->external_nlba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) super->internal_lbasize = cpu_to_le32(arena->internal_lbasize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) super->internal_nlba = cpu_to_le32(arena->internal_nlba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) super->nfree = cpu_to_le32(arena->nfree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) super->infosize = cpu_to_le32(sizeof(struct btt_sb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) super->nextoff = cpu_to_le64(arena->nextoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) * Subtract arena->infooff (arena start) so numbers are relative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) * to 'this' arena
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) super->dataoff = cpu_to_le64(arena->dataoff - arena->infooff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) super->mapoff = cpu_to_le64(arena->mapoff - arena->infooff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) super->logoff = cpu_to_le64(arena->logoff - arena->infooff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) super->info2off = cpu_to_le64(arena->info2off - arena->infooff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) super->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) sum = nd_sb_checksum((struct nd_gen_sb *) super);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) super->checksum = cpu_to_le64(sum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) ret = btt_info_write(arena, super);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) kfree(super);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) * This function completes the initialization for the BTT namespace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) * such that it is ready to accept IOs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) static int btt_meta_init(struct btt *btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) struct arena_info *arena;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) mutex_lock(&btt->init_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) list_for_each_entry(arena, &btt->arena_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) ret = btt_arena_write_layout(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) ret = btt_freelist_init(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) ret = btt_rtt_init(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) ret = btt_maplocks_init(arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) btt->init_state = INIT_READY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) mutex_unlock(&btt->init_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) static u32 btt_meta_size(struct btt *btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) return btt->lbasize - btt->sector_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) * This function calculates the arena in which the given LBA lies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) * by doing a linear walk. This is acceptable since we expect only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) * a few arenas. If we have backing devices that get much larger,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) * we can construct a balanced binary tree of arenas at init time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) * so that this range search becomes faster.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) static int lba_to_arena(struct btt *btt, sector_t sector, __u32 *premap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) struct arena_info **arena)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) struct arena_info *arena_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) __u64 lba = div_u64(sector << SECTOR_SHIFT, btt->sector_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) list_for_each_entry(arena_list, &btt->arena_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) if (lba < arena_list->external_nlba) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) *arena = arena_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) *premap = lba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) lba -= arena_list->external_nlba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) * The following (lock_map, unlock_map) are mostly just to improve
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * readability, since they index into an array of locks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) static void lock_map(struct arena_info *arena, u32 premap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) __acquires(&arena->map_locks[idx].lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) u32 idx = (premap * MAP_ENT_SIZE / L1_CACHE_BYTES) % arena->nfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) spin_lock(&arena->map_locks[idx].lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) static void unlock_map(struct arena_info *arena, u32 premap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) __releases(&arena->map_locks[idx].lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) u32 idx = (premap * MAP_ENT_SIZE / L1_CACHE_BYTES) % arena->nfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) spin_unlock(&arena->map_locks[idx].lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) static int btt_data_read(struct arena_info *arena, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) unsigned int off, u32 lba, u32 len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) u64 nsoff = to_namespace_offset(arena, lba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) void *mem = kmap_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) ret = arena_read_bytes(arena, nsoff, mem + off, len, NVDIMM_IO_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) kunmap_atomic(mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) static int btt_data_write(struct arena_info *arena, u32 lba,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) struct page *page, unsigned int off, u32 len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) u64 nsoff = to_namespace_offset(arena, lba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) void *mem = kmap_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) ret = arena_write_bytes(arena, nsoff, mem + off, len, NVDIMM_IO_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) kunmap_atomic(mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) static void zero_fill_data(struct page *page, unsigned int off, u32 len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) void *mem = kmap_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) memset(mem + off, 0, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) kunmap_atomic(mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) #ifdef CONFIG_BLK_DEV_INTEGRITY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) static int btt_rw_integrity(struct btt *btt, struct bio_integrity_payload *bip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) struct arena_info *arena, u32 postmap, int rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) unsigned int len = btt_meta_size(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) u64 meta_nsoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) if (bip == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) meta_nsoff = to_namespace_offset(arena, postmap) + btt->sector_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) while (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) unsigned int cur_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) struct bio_vec bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) void *mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) bv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) * The 'bv' obtained from bvec_iter_bvec has its .bv_len and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) * .bv_offset already adjusted for iter->bi_bvec_done, and we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) * can use those directly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) cur_len = min(len, bv.bv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) mem = kmap_atomic(bv.bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) ret = arena_write_bytes(arena, meta_nsoff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) mem + bv.bv_offset, cur_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) NVDIMM_IO_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) ret = arena_read_bytes(arena, meta_nsoff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) mem + bv.bv_offset, cur_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) NVDIMM_IO_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) kunmap_atomic(mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) len -= cur_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) meta_nsoff += cur_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) if (!bvec_iter_advance(bip->bip_vec, &bip->bip_iter, cur_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) #else /* CONFIG_BLK_DEV_INTEGRITY */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) static int btt_rw_integrity(struct btt *btt, struct bio_integrity_payload *bip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) struct arena_info *arena, u32 postmap, int rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) static int btt_read_pg(struct btt *btt, struct bio_integrity_payload *bip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) struct page *page, unsigned int off, sector_t sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) int t_flag, e_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) struct arena_info *arena = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) u32 lane = 0, premap, postmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) while (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) u32 cur_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) lane = nd_region_acquire_lane(btt->nd_region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) ret = lba_to_arena(btt, sector, &premap, &arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) goto out_lane;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) cur_len = min(btt->sector_size, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) ret = btt_map_read(arena, premap, &postmap, &t_flag, &e_flag,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) NVDIMM_IO_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) goto out_lane;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) * We loop to make sure that the post map LBA didn't change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) * from under us between writing the RTT and doing the actual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) * read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) u32 new_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) int new_t, new_e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) if (t_flag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) zero_fill_data(page, off, cur_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) goto out_lane;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) if (e_flag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) goto out_lane;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) arena->rtt[lane] = RTT_VALID | postmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) * Barrier to make sure this write is not reordered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) * to do the verification map_read before the RTT store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) ret = btt_map_read(arena, premap, &new_map, &new_t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) &new_e, NVDIMM_IO_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) goto out_rtt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) if ((postmap == new_map) && (t_flag == new_t) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) (e_flag == new_e))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) postmap = new_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) t_flag = new_t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) e_flag = new_e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) ret = btt_data_read(arena, page, off, postmap, cur_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) /* Media error - set the e_flag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) if (btt_map_write(arena, premap, postmap, 0, 1, NVDIMM_IO_ATOMIC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) dev_warn_ratelimited(to_dev(arena),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) "Error persistently tracking bad blocks at %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) premap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) goto out_rtt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) if (bip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) ret = btt_rw_integrity(btt, bip, arena, postmap, READ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) goto out_rtt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) arena->rtt[lane] = RTT_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) nd_region_release_lane(btt->nd_region, lane);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) len -= cur_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) off += cur_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) sector += btt->sector_size >> SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) out_rtt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) arena->rtt[lane] = RTT_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) out_lane:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) nd_region_release_lane(btt->nd_region, lane);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * Normally, arena_{read,write}_bytes will take care of the initial offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) * adjustment, but in the case of btt_is_badblock, where we query is_bad_pmem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) * we need the final, raw namespace offset here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) static bool btt_is_badblock(struct btt *btt, struct arena_info *arena,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) u32 postmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) u64 nsoff = adjust_initial_offset(arena->nd_btt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) to_namespace_offset(arena, postmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) sector_t phys_sector = nsoff >> 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) return is_bad_pmem(btt->phys_bb, phys_sector, arena->internal_lbasize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) sector_t sector, struct page *page, unsigned int off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) struct arena_info *arena = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) u32 premap = 0, old_postmap, new_postmap, lane = 0, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) struct log_entry log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) int sub;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) while (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) u32 cur_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) int e_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) lane = nd_region_acquire_lane(btt->nd_region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) ret = lba_to_arena(btt, sector, &premap, &arena);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) goto out_lane;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) cur_len = min(btt->sector_size, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) if ((arena->flags & IB_FLAG_ERROR_MASK) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) goto out_lane;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) if (btt_is_badblock(btt, arena, arena->freelist[lane].block))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) arena->freelist[lane].has_err = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) if (mutex_is_locked(&arena->err_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) || arena->freelist[lane].has_err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) nd_region_release_lane(btt->nd_region, lane);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) ret = arena_clear_freelist_error(arena, lane);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) /* OK to acquire a different lane/free block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) new_postmap = arena->freelist[lane].block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) /* Wait if the new block is being read from */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) for (i = 0; i < arena->nfree; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) while (arena->rtt[i] == (RTT_VALID | new_postmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) if (new_postmap >= arena->internal_nlba) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) goto out_lane;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) ret = btt_data_write(arena, new_postmap, page, off, cur_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) goto out_lane;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) if (bip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) ret = btt_rw_integrity(btt, bip, arena, new_postmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) goto out_lane;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) lock_map(arena, premap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) ret = btt_map_read(arena, premap, &old_postmap, NULL, &e_flag,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) NVDIMM_IO_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) goto out_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) if (old_postmap >= arena->internal_nlba) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) goto out_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) if (e_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) set_e_flag(old_postmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) log.lba = cpu_to_le32(premap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) log.old_map = cpu_to_le32(old_postmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) log.new_map = cpu_to_le32(new_postmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) log.seq = cpu_to_le32(arena->freelist[lane].seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) sub = arena->freelist[lane].sub;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) ret = btt_flog_write(arena, lane, sub, &log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) goto out_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) ret = btt_map_write(arena, premap, new_postmap, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) NVDIMM_IO_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) goto out_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) unlock_map(arena, premap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) nd_region_release_lane(btt->nd_region, lane);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) if (e_flag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) ret = arena_clear_freelist_error(arena, lane);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) len -= cur_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) off += cur_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) sector += btt->sector_size >> SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) out_map:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) unlock_map(arena, premap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) out_lane:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) nd_region_release_lane(btt->nd_region, lane);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) struct page *page, unsigned int len, unsigned int off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) unsigned int op, sector_t sector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) if (!op_is_write(op)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) ret = btt_read_pg(btt, bip, page, off, sector, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) flush_dcache_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) flush_dcache_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) ret = btt_write_pg(btt, bip, sector, page, off, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) static blk_qc_t btt_submit_bio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) struct bio_integrity_payload *bip = bio_integrity(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) struct btt *btt = bio->bi_disk->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) struct bvec_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) unsigned long start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) struct bio_vec bvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) bool do_acct;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) if (!bio_integrity_prep(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) do_acct = blk_queue_io_stat(bio->bi_disk->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) if (do_acct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) start = bio_start_io_acct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) bio_for_each_segment(bvec, bio, iter) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) unsigned int len = bvec.bv_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) if (len > PAGE_SIZE || len < btt->sector_size ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) len % btt->sector_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) dev_err_ratelimited(&btt->nd_btt->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) "unaligned bio segment (len: %d)\n", len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) bio->bi_status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) err = btt_do_bvec(btt, bip, bvec.bv_page, len, bvec.bv_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) bio_op(bio), iter.bi_sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) dev_err(&btt->nd_btt->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) "io error in %s sector %lld, len %d,\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) (op_is_write(bio_op(bio))) ? "WRITE" :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) "READ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) (unsigned long long) iter.bi_sector, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) bio->bi_status = errno_to_blk_status(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) if (do_acct)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) bio_end_io_acct(bio, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) return BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) static int btt_rw_page(struct block_device *bdev, sector_t sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) struct page *page, unsigned int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) struct btt *btt = bdev->bd_disk->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) rc = btt_do_bvec(btt, NULL, page, thp_size(page), 0, op, sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) if (rc == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) page_endio(page, op_is_write(op), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) static int btt_getgeo(struct block_device *bd, struct hd_geometry *geo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) /* some standard values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) geo->heads = 1 << 6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) geo->sectors = 1 << 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) geo->cylinders = get_capacity(bd->bd_disk) >> 11;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) static const struct block_device_operations btt_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) .submit_bio = btt_submit_bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) .rw_page = btt_rw_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) .getgeo = btt_getgeo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) static int btt_blk_init(struct btt *btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) struct nd_btt *nd_btt = btt->nd_btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) struct nd_namespace_common *ndns = nd_btt->ndns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) /* create a new disk and request queue for btt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) btt->btt_queue = blk_alloc_queue(NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) if (!btt->btt_queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) btt->btt_disk = alloc_disk(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) if (!btt->btt_disk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) blk_cleanup_queue(btt->btt_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) nvdimm_namespace_disk_name(ndns, btt->btt_disk->disk_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) btt->btt_disk->first_minor = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) btt->btt_disk->fops = &btt_fops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) btt->btt_disk->private_data = btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) btt->btt_disk->queue = btt->btt_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) btt->btt_disk->flags = GENHD_FL_EXT_DEVT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) blk_queue_logical_block_size(btt->btt_queue, btt->sector_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) blk_queue_max_hw_sectors(btt->btt_queue, UINT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) blk_queue_flag_set(QUEUE_FLAG_NONROT, btt->btt_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) if (btt_meta_size(btt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) int rc = nd_integrity_init(btt->btt_disk, btt_meta_size(btt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) del_gendisk(btt->btt_disk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) put_disk(btt->btt_disk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) blk_cleanup_queue(btt->btt_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) set_capacity(btt->btt_disk, btt->nlba * btt->sector_size >> 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) device_add_disk(&btt->nd_btt->dev, btt->btt_disk, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) btt->nd_btt->size = btt->nlba * (u64)btt->sector_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) nvdimm_check_and_set_ro(btt->btt_disk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) static void btt_blk_cleanup(struct btt *btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) del_gendisk(btt->btt_disk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) put_disk(btt->btt_disk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) blk_cleanup_queue(btt->btt_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) * btt_init - initialize a block translation table for the given device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) * @nd_btt: device with BTT geometry and backing device info
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) * @rawsize: raw size in bytes of the backing device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) * @lbasize: lba size of the backing device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) * @uuid: A uuid for the backing device - this is stored on media
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) * @maxlane: maximum number of parallel requests the device can handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) * Initialize a Block Translation Table on a backing device to provide
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) * single sector power fail atomicity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) * Context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) * Might sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) * Pointer to a new struct btt on success, NULL on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) u32 lbasize, u8 *uuid, struct nd_region *nd_region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) struct btt *btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) struct nd_namespace_io *nsio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) struct device *dev = &nd_btt->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) btt = devm_kzalloc(dev, sizeof(struct btt), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) if (!btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) btt->nd_btt = nd_btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) btt->rawsize = rawsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) btt->lbasize = lbasize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) btt->sector_size = ((lbasize >= 4096) ? 4096 : 512);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) INIT_LIST_HEAD(&btt->arena_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) mutex_init(&btt->init_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) btt->nd_region = nd_region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) nsio = to_nd_namespace_io(&nd_btt->ndns->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) btt->phys_bb = &nsio->bb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) ret = discover_arenas(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) dev_err(dev, "init: error in arena_discover: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) if (btt->init_state != INIT_READY && nd_region->ro) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) dev_warn(dev, "%s is read-only, unable to init btt metadata\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) dev_name(&nd_region->dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) } else if (btt->init_state != INIT_READY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) btt->num_arenas = (rawsize / ARENA_MAX_SIZE) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) ((rawsize % ARENA_MAX_SIZE) ? 1 : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) dev_dbg(dev, "init: %d arenas for %llu rawsize\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) btt->num_arenas, rawsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) ret = create_arenas(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) dev_info(dev, "init: create_arenas: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) ret = btt_meta_init(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) dev_err(dev, "init: error in meta_init: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) ret = btt_blk_init(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) dev_err(dev, "init: error in blk_init: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) btt_debugfs_init(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) return btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) * btt_fini - de-initialize a BTT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) * @btt: the BTT handle that was generated by btt_init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) * De-initialize a Block Translation Table on device removal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) * Context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) * Might sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) static void btt_fini(struct btt *btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) if (btt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) btt_blk_cleanup(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) free_arenas(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) debugfs_remove_recursive(btt->debugfs_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) int nvdimm_namespace_attach_btt(struct nd_namespace_common *ndns)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) struct nd_btt *nd_btt = to_nd_btt(ndns->claim);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) struct nd_region *nd_region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) struct btt_sb *btt_sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) struct btt *btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) size_t size, rawsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) if (!nd_btt->uuid || !nd_btt->ndns || !nd_btt->lbasize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) dev_dbg(&nd_btt->dev, "incomplete btt configuration\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) btt_sb = devm_kzalloc(&nd_btt->dev, sizeof(*btt_sb), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) if (!btt_sb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) size = nvdimm_namespace_capacity(ndns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) rc = devm_namespace_enable(&nd_btt->dev, ndns, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) * If this returns < 0, that is ok as it just means there wasn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) * an existing BTT, and we're creating a new one. We still need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) * call this as we need the version dependent fields in nd_btt to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) * set correctly based on the holder class
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) nd_btt_version(nd_btt, ndns, btt_sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) rawsize = size - nd_btt->initial_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) if (rawsize < ARENA_MIN_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) dev_dbg(&nd_btt->dev, "%s must be at least %ld bytes\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) dev_name(&ndns->dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) ARENA_MIN_SIZE + nd_btt->initial_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) nd_region = to_nd_region(nd_btt->dev.parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) btt = btt_init(nd_btt, rawsize, nd_btt->lbasize, nd_btt->uuid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) nd_region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) if (!btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) nd_btt->btt = btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) EXPORT_SYMBOL(nvdimm_namespace_attach_btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) int nvdimm_namespace_detach_btt(struct nd_btt *nd_btt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) struct btt *btt = nd_btt->btt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) btt_fini(btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) nd_btt->btt = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) EXPORT_SYMBOL(nvdimm_namespace_detach_btt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) static int __init nd_btt_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) debugfs_root = debugfs_create_dir("btt", NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) if (IS_ERR_OR_NULL(debugfs_root))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) rc = -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) static void __exit nd_btt_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) debugfs_remove_recursive(debugfs_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) MODULE_ALIAS_ND_DEVICE(ND_DEVICE_BTT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) MODULE_AUTHOR("Vishal Verma <vishal.l.verma@linux.intel.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) module_init(nd_btt_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) module_exit(nd_btt_exit);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags