^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2015 IT University of Copenhagen (rrpc.h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2016 CNEX Labs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Initial release: Matias Bjorling <matias@cnexlabs.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Write buffering: Javier Gonzalez <javier@cnexlabs.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * This program is free software; you can redistribute it and/or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * modify it under the terms of the GNU General Public License version
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * 2 as published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * Implementation of a Physical Block-device target for Open-channel SSDs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #ifndef PBLK_H_
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define PBLK_H_
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/blk-mq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/bio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/crc32.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/uuid.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/lightnvm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /* Run only GC if less than 1/X blocks are free */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define GC_LIMIT_INVERSE 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define GC_TIME_MSECS 1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define PBLK_SECTOR (512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define PBLK_EXPOSED_PAGE_SIZE (4096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define PBLK_NR_CLOSE_JOBS (4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define PBLK_CACHE_NAME_LEN (DISK_NAME_LEN + 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /* Max 512 LUNs per device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define PBLK_MAX_LUNS_BITMAP (4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define NR_PHY_IN_LOG (PBLK_EXPOSED_PAGE_SIZE / PBLK_SECTOR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /* Static pool sizes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define PBLK_GEN_WS_POOL_SIZE (2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define PBLK_DEFAULT_OP (11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) PBLK_READ = READ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) PBLK_WRITE = WRITE,/* Write from write buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) PBLK_WRITE_INT, /* Internal write - no write buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) PBLK_READ_RECOV, /* Recovery read - errors allowed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) PBLK_ERASE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /* IO Types */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) PBLK_IOTYPE_USER = 1 << 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) PBLK_IOTYPE_GC = 1 << 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /* Write buffer flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) PBLK_FLUSH_ENTRY = 1 << 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) PBLK_WRITTEN_DATA = 1 << 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) PBLK_SUBMITTED_ENTRY = 1 << 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) PBLK_WRITABLE_ENTRY = 1 << 5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) PBLK_BLK_ST_OPEN = 0x1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) PBLK_BLK_ST_CLOSED = 0x2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) PBLK_CHUNK_RESET_START,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) PBLK_CHUNK_RESET_DONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) PBLK_CHUNK_RESET_FAILED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct pblk_sec_meta {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) u64 reserved;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) __le64 lba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) /* The number of GC lists and the rate-limiter states go together. This way the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * rate-limiter can dictate how much GC is needed based on resource utilization.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define PBLK_GC_NR_LISTS 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) PBLK_RL_OFF = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) PBLK_RL_WERR = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) PBLK_RL_HIGH = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) PBLK_RL_MID = 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) PBLK_RL_LOW = 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define pblk_dma_ppa_size (sizeof(u64) * NVM_MAX_VLBA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* write buffer completion context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct pblk_c_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) struct list_head list; /* Head for out-of-order completion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) unsigned long *lun_bitmap; /* Luns used on current request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) unsigned int sentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) unsigned int nr_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) unsigned int nr_padded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /* read context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct pblk_g_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) void *private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) unsigned long start_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) u64 lba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /* Pad context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct pblk_pad_rq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct pblk *pblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) struct completion wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) struct kref ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /* Recovery context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct pblk_rec_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct pblk *pblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) struct nvm_rq *rqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) struct work_struct ws_rec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /* Write context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct pblk_w_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct bio_list bios; /* Original bios - used for completion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * in REQ_FUA, REQ_FLUSH case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) u64 lba; /* Logic addr. associated with entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) struct ppa_addr ppa; /* Physic addr. associated with entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) int flags; /* Write context flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct pblk_rb_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) struct ppa_addr cacheline; /* Cacheline for this entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) void *data; /* Pointer to data on this entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) struct pblk_w_ctx w_ctx; /* Context for this entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) struct list_head index; /* List head to enable indexes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #define EMPTY_ENTRY (~0U)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) struct pblk_rb_pages {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct page *pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) int order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct pblk_rb {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) struct pblk_rb_entry *entries; /* Ring buffer entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) unsigned int mem; /* Write offset - points to next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * writable entry in memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) unsigned int subm; /* Read offset - points to last entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * that has been submitted to the media
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * to be persisted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) unsigned int sync; /* Synced - backpointer that signals
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * the last submitted entry that has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * been successfully persisted to media
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) unsigned int flush_point; /* Sync point - last entry that must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * flushed to the media. Used with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * REQ_FLUSH and REQ_FUA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) unsigned int l2p_update; /* l2p update point - next entry for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * which l2p mapping will be updated to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * contain a device ppa address (instead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * of a cacheline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) unsigned int nr_entries; /* Number of entries in write buffer -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * must be a power of two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) unsigned int seg_size; /* Size of the data segments being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * stored on each entry. Typically this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * will be 4KB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) unsigned int back_thres; /* Threshold that shall be maintained by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * the backpointer in order to respect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * geo->mw_cunits on a per chunk basis
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct list_head pages; /* List of data pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) spinlock_t w_lock; /* Write lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) spinlock_t s_lock; /* Sync lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) atomic_t inflight_flush_point; /* Not served REQ_FLUSH | REQ_FUA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) #define PBLK_RECOVERY_SECTORS 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct pblk_lun {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) struct ppa_addr bppa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) struct semaphore wr_sem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) struct pblk_gc_rq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) struct pblk_line *line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) void *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) u64 paddr_list[NVM_MAX_VLBA];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) u64 lba_list[NVM_MAX_VLBA];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) int nr_secs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) int secs_to_gc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct pblk_gc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) /* These states are not protected by a lock since (i) they are in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * fast path, and (ii) they are not critical.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) int gc_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) int gc_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) int gc_forced;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct task_struct *gc_ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct task_struct *gc_writer_ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct task_struct *gc_reader_ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) struct workqueue_struct *gc_line_reader_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) struct workqueue_struct *gc_reader_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) struct timer_list gc_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) struct semaphore gc_sem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) atomic_t read_inflight_gc; /* Number of lines with inflight GC reads */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) atomic_t pipeline_gc; /* Number of lines in the GC pipeline -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * started reads to finished writes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) int w_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct list_head w_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct list_head r_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) spinlock_t w_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) spinlock_t r_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct pblk_rl {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) unsigned int high; /* Upper threshold for rate limiter (free run -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * user I/O rate limiter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) unsigned int high_pw; /* High rounded up as a power of 2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) #define PBLK_USER_HIGH_THRS 8 /* Begin write limit at 12% available blks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) #define PBLK_USER_LOW_THRS 10 /* Aggressive GC at 10% available blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) int rb_windows_pw; /* Number of rate windows in the write buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * given as a power-of-2. This guarantees that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * when user I/O is being rate limited, there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * will be reserved enough space for the GC to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * place its payload. A window is of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * pblk->max_write_pgs size, which in NVMe is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * 64, i.e., 256kb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) int rb_budget; /* Total number of entries available for I/O */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) int rb_user_max; /* Max buffer entries available for user I/O */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) int rb_gc_max; /* Max buffer entries available for GC I/O */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) int rb_gc_rsv; /* Reserved buffer entries for GC I/O */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) int rb_state; /* Rate-limiter current state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) int rb_max_io; /* Maximum size for an I/O giving the config */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) atomic_t rb_user_cnt; /* User I/O buffer counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) atomic_t rb_gc_cnt; /* GC I/O buffer counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) atomic_t rb_space; /* Space limit in case of reaching capacity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) int rsv_blocks; /* Reserved blocks for GC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) int rb_user_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) int rb_gc_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) atomic_t werr_lines; /* Number of write error lines that needs gc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) struct timer_list u_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned long total_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) atomic_t free_blocks; /* Total number of free blocks (+ OP) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) atomic_t free_user_blocks; /* Number of user free blocks (no OP) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) #define PBLK_LINE_EMPTY (~0U)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /* Line Types */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) PBLK_LINETYPE_FREE = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) PBLK_LINETYPE_LOG = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) PBLK_LINETYPE_DATA = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* Line state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) PBLK_LINESTATE_NEW = 9,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) PBLK_LINESTATE_FREE = 10,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) PBLK_LINESTATE_OPEN = 11,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) PBLK_LINESTATE_CLOSED = 12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) PBLK_LINESTATE_GC = 13,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) PBLK_LINESTATE_BAD = 14,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) PBLK_LINESTATE_CORRUPT = 15,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) /* GC group */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) PBLK_LINEGC_NONE = 20,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) PBLK_LINEGC_EMPTY = 21,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) PBLK_LINEGC_LOW = 22,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) PBLK_LINEGC_MID = 23,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) PBLK_LINEGC_HIGH = 24,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) PBLK_LINEGC_FULL = 25,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) PBLK_LINEGC_WERR = 26
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) #define PBLK_MAGIC 0x70626c6b /*pblk*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) /* emeta/smeta persistent storage format versions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * Changes in major version requires offline migration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * Changes in minor version are handled automatically during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * recovery.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) #define SMETA_VERSION_MAJOR (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) #define SMETA_VERSION_MINOR (1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) #define EMETA_VERSION_MAJOR (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) #define EMETA_VERSION_MINOR (2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) struct line_header {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) __le32 crc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) __le32 identifier; /* pblk identifier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) __u8 uuid[16]; /* instance uuid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) __le16 type; /* line type */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) __u8 version_major; /* version major */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) __u8 version_minor; /* version minor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) __le32 id; /* line id for current line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct line_smeta {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) struct line_header header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) __le32 crc; /* Full structure including struct crc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) /* Previous line metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) __le32 prev_id; /* Line id for previous line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /* Current line metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) __le64 seq_nr; /* Sequence number for current line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /* Active writers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) __le32 window_wr_lun; /* Number of parallel LUNs to write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) __le32 rsvd[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) __le64 lun_bitmap[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * Metadata layout in media:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * First sector:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * 1. struct line_emeta
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * 2. bad block bitmap (u64 * window_wr_lun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * 3. write amplification counters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * Mid sectors (start at lbas_sector):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * 3. nr_lbas (u64) forming lba list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * Last sectors (start at vsc_sector):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * 4. u32 valid sector count (vsc) for all lines (~0U: free line)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct line_emeta {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) struct line_header header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) __le32 crc; /* Full structure including struct crc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) /* Previous line metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) __le32 prev_id; /* Line id for prev line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /* Current line metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) __le64 seq_nr; /* Sequence number for current line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) /* Active writers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) __le32 window_wr_lun; /* Number of parallel LUNs to write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) /* Bookkeeping for recovery */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) __le32 next_id; /* Line id for next line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) __le64 nr_lbas; /* Number of lbas mapped in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) __le64 nr_valid_lbas; /* Number of valid lbas mapped in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) __le64 bb_bitmap[]; /* Updated bad block bitmap for line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /* Write amplification counters stored on media */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) struct wa_counters {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) __le64 user; /* Number of user written sectors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) __le64 gc; /* Number of sectors written by GC*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) __le64 pad; /* Number of padded sectors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) struct pblk_emeta {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) struct line_emeta *buf; /* emeta buffer in media format */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) int mem; /* Write offset - points to next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * writable entry in memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) atomic_t sync; /* Synced - backpointer that signals the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * last entry that has been successfully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * persisted to media
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) unsigned int nr_entries; /* Number of emeta entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) struct pblk_smeta {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) struct line_smeta *buf; /* smeta buffer in persistent format */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) struct pblk_w_err_gc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) int has_write_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) int has_gc_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) __le64 *lba_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) struct pblk_line {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) struct pblk *pblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) unsigned int id; /* Line number corresponds to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * block line
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) unsigned int seq_nr; /* Unique line sequence number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) int state; /* PBLK_LINESTATE_X */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) int type; /* PBLK_LINETYPE_X */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) int gc_group; /* PBLK_LINEGC_X */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct list_head list; /* Free, GC lists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) unsigned long *lun_bitmap; /* Bitmap for LUNs mapped in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) struct nvm_chk_meta *chks; /* Chunks forming line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) struct pblk_smeta *smeta; /* Start metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) struct pblk_emeta *emeta; /* End medatada */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) int meta_line; /* Metadata line id */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) int meta_distance; /* Distance between data and metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) u64 emeta_ssec; /* Sector where emeta starts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) unsigned int sec_in_line; /* Number of usable secs in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) atomic_t blk_in_line; /* Number of good blocks in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) unsigned long *blk_bitmap; /* Bitmap for valid/invalid blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) unsigned long *erase_bitmap; /* Bitmap for erased blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) unsigned long *map_bitmap; /* Bitmap for mapped sectors in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) unsigned long *invalid_bitmap; /* Bitmap for invalid sectors in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) atomic_t left_eblks; /* Blocks left for erasing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) atomic_t left_seblks; /* Blocks left for sync erasing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) int left_msecs; /* Sectors left for mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) unsigned int cur_sec; /* Sector map pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) unsigned int nr_valid_lbas; /* Number of valid lbas in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) __le32 *vsc; /* Valid sector count in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) struct kref ref; /* Write buffer L2P references */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) atomic_t sec_to_update; /* Outstanding L2P updates to ppa */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) struct pblk_w_err_gc *w_err_gc; /* Write error gc recovery metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) spinlock_t lock; /* Necessary for invalid_bitmap only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) #define PBLK_DATA_LINES 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) PBLK_EMETA_TYPE_HEADER = 1, /* struct line_emeta first sector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) PBLK_EMETA_TYPE_LLBA = 2, /* lba list - type: __le64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) PBLK_EMETA_TYPE_VSC = 3, /* vsc list - type: __le32 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) struct pblk_line_mgmt {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) int nr_lines; /* Total number of full lines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) int nr_free_lines; /* Number of full lines in free list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) /* Free lists - use free_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) struct list_head free_list; /* Full lines ready to use */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) struct list_head corrupt_list; /* Full lines corrupted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) struct list_head bad_list; /* Full lines bad */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) /* GC lists - use gc_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) struct list_head *gc_lists[PBLK_GC_NR_LISTS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) struct list_head gc_high_list; /* Full lines ready to GC, high isc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) struct list_head gc_mid_list; /* Full lines ready to GC, mid isc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) struct list_head gc_low_list; /* Full lines ready to GC, low isc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) struct list_head gc_werr_list; /* Write err recovery list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) struct list_head gc_full_list; /* Full lines ready to GC, no valid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) struct list_head gc_empty_list; /* Full lines close, all valid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) struct pblk_line *log_line; /* Current FTL log line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) struct pblk_line *data_line; /* Current data line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) struct pblk_line *log_next; /* Next FTL log line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) struct pblk_line *data_next; /* Next data line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct list_head emeta_list; /* Lines queued to schedule emeta */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) __le32 *vsc_list; /* Valid sector counts for all lines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /* Pre-allocated metadata for data lines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) struct pblk_smeta *sline_meta[PBLK_DATA_LINES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) struct pblk_emeta *eline_meta[PBLK_DATA_LINES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) unsigned long meta_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /* Cache and mempool for map/invalid bitmaps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) struct kmem_cache *bitmap_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) mempool_t *bitmap_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) /* Helpers for fast bitmap calculations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) unsigned long *bb_template;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) unsigned long *bb_aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) unsigned long d_seq_nr; /* Data line unique sequence number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) unsigned long l_seq_nr; /* Log line unique sequence number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) spinlock_t free_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) spinlock_t close_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) spinlock_t gc_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) struct pblk_line_meta {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) unsigned int smeta_len; /* Total length for smeta */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) unsigned int smeta_sec; /* Sectors needed for smeta */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) unsigned int emeta_len[4]; /* Lengths for emeta:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * [0]: Total
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * [1]: struct line_emeta +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) * bb_bitmap + struct wa_counters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * [2]: L2P portion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * [3]: vsc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) unsigned int emeta_sec[4]; /* Sectors needed for emeta. Same layout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * as emeta_len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) unsigned int emeta_bb; /* Boundary for bb that affects emeta */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) unsigned int vsc_list_len; /* Length for vsc list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) unsigned int sec_bitmap_len; /* Length for sector bitmap in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) unsigned int blk_bitmap_len; /* Length for block bitmap in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) unsigned int lun_bitmap_len; /* Length for lun bitmap in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) unsigned int blk_per_line; /* Number of blocks in a full line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) unsigned int sec_per_line; /* Number of sectors in a line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) unsigned int dsec_per_line; /* Number of data sectors in a line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) unsigned int min_blk_line; /* Min. number of good blocks in line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) unsigned int mid_thrs; /* Threshold for GC mid list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) unsigned int high_thrs; /* Threshold for GC high list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) unsigned int meta_distance; /* Distance between data and metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) PBLK_STATE_RUNNING = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) PBLK_STATE_STOPPING = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) PBLK_STATE_RECOVERING = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) PBLK_STATE_STOPPED = 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) /* Internal format to support not power-of-2 device formats */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) struct pblk_addrf {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /* gen to dev */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) int sec_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) int ch_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) int lun_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /* dev to gen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) int sec_lun_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) int sec_ws_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) struct pblk {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) struct nvm_tgt_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) struct gendisk *disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) struct kobject kobj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) struct pblk_lun *luns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) struct pblk_line *lines; /* Line array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) struct pblk_line_mgmt l_mg; /* Line management */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) struct pblk_line_meta lm; /* Line metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) struct nvm_addrf addrf; /* Aligned address format */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) struct pblk_addrf uaddrf; /* Unaligned address format */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) int addrf_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) struct pblk_rb rwb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) int state; /* pblk line state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) int min_write_pgs; /* Minimum amount of pages required by controller */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) int min_write_pgs_data; /* Minimum amount of payload pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) int max_write_pgs; /* Maximum amount of pages supported by controller */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) int oob_meta_size; /* Size of OOB sector metadata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) sector_t capacity; /* Device capacity when bad blocks are subtracted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) int op; /* Percentage of device used for over-provisioning */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) int op_blks; /* Number of blocks used for over-provisioning */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) /* pblk provisioning values. Used by rate limiter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) struct pblk_rl rl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) int sec_per_write;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) guid_t instance_uuid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) /* Persistent write amplification counters, 4kb sector I/Os */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) atomic64_t user_wa; /* Sectors written by user */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) atomic64_t gc_wa; /* Sectors written by GC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) atomic64_t pad_wa; /* Padded sectors written */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) /* Reset values for delta write amplification measurements */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) u64 user_rst_wa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) u64 gc_rst_wa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) u64 pad_rst_wa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) /* Counters used for calculating padding distribution */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) atomic64_t *pad_dist; /* Padding distribution buckets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) u64 nr_flush_rst; /* Flushes reset value for pad dist.*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) atomic64_t nr_flush; /* Number of flush/fua I/O */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) /* Non-persistent debug counters, 4kb sector I/Os */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) atomic_long_t inflight_writes; /* Inflight writes (user and gc) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) atomic_long_t padded_writes; /* Sectors padded due to flush/fua */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) atomic_long_t padded_wb; /* Sectors padded in write buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) atomic_long_t req_writes; /* Sectors stored on write buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) atomic_long_t sub_writes; /* Sectors submitted from buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) atomic_long_t sync_writes; /* Sectors synced to media */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) atomic_long_t inflight_reads; /* Inflight sector read requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) atomic_long_t cache_reads; /* Read requests that hit the cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) atomic_long_t sync_reads; /* Completed sector read requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) atomic_long_t recov_writes; /* Sectors submitted from recovery */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) atomic_long_t recov_gc_writes; /* Sectors submitted from write GC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) atomic_long_t recov_gc_reads; /* Sectors submitted from read GC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) atomic_long_t read_failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) atomic_long_t read_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) atomic_long_t read_high_ecc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) atomic_long_t read_failed_gc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) atomic_long_t write_failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) atomic_long_t erase_failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) atomic_t inflight_io; /* General inflight I/O counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) struct task_struct *writer_ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) /* Simple translation map of logical addresses to physical addresses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * The logical addresses is known by the host system, while the physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) * addresses are used when writing to the disk block device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) unsigned char *trans_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) spinlock_t trans_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) struct list_head compl_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) spinlock_t resubmit_lock; /* Resubmit list lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) struct list_head resubmit_list; /* Resubmit list for failed writes*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) mempool_t page_bio_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) mempool_t gen_ws_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) mempool_t rec_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) mempool_t r_rq_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) mempool_t w_rq_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) mempool_t e_rq_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) struct workqueue_struct *close_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) struct workqueue_struct *bb_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) struct workqueue_struct *r_end_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) struct timer_list wtimer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) struct pblk_gc gc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) struct pblk_line_ws {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) struct pblk *pblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) struct pblk_line *line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) void *priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) struct work_struct ws;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) #define pblk_g_rq_size (sizeof(struct nvm_rq) + sizeof(struct pblk_g_ctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) #define pblk_w_rq_size (sizeof(struct nvm_rq) + sizeof(struct pblk_c_ctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) #define pblk_err(pblk, fmt, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) pr_err("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) #define pblk_info(pblk, fmt, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) pr_info("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) #define pblk_warn(pblk, fmt, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) pr_warn("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) #define pblk_debug(pblk, fmt, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) pr_debug("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) * pblk ring buffer operations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) int pblk_rb_init(struct pblk_rb *rb, unsigned int size, unsigned int threshold,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) unsigned int seg_sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) unsigned int nr_entries, unsigned int *pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) unsigned int *pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) void pblk_rb_write_entry_user(struct pblk_rb *rb, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) struct pblk_w_ctx w_ctx, unsigned int pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) struct pblk_w_ctx w_ctx, struct pblk_line *line,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) u64 paddr, unsigned int pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) void pblk_rb_flush(struct pblk_rb *rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) void pblk_rb_sync_l2p(struct pblk_rb *rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) unsigned int pos, unsigned int nr_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) unsigned int count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) int pblk_rb_copy_to_bio(struct pblk_rb *rb, struct bio *bio, sector_t lba,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) struct ppa_addr ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) unsigned int pblk_rb_ptr_wrap(struct pblk_rb *rb, unsigned int p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) unsigned int nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) unsigned int pblk_rb_read_count(struct pblk_rb *rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) unsigned int pblk_rb_sync_count(struct pblk_rb *rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) int pblk_rb_tear_down_check(struct pblk_rb *rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) void pblk_rb_free(struct pblk_rb *rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * pblk core
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) int pblk_alloc_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) void pblk_free_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) void pblk_set_sec_per_write(struct pblk *pblk, int sec_per_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) int pblk_setup_w_rec_rq(struct pblk *pblk, struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) struct pblk_c_ctx *c_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) void pblk_discard(struct pblk *pblk, struct bio *bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) struct nvm_chk_meta *pblk_get_chunk_meta(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) struct nvm_chk_meta *pblk_chunk_get_off(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) struct nvm_chk_meta *lp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) struct ppa_addr ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) void pblk_log_write_err(struct pblk *pblk, struct nvm_rq *rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) void pblk_log_read_err(struct pblk *pblk, struct nvm_rq *rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd, void *buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd, void *buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) void pblk_check_chunk_state_update(struct pblk *pblk, struct nvm_rq *rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) struct pblk_line *pblk_line_get(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) struct pblk_line *pblk_line_get_first_data(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) struct pblk_line *pblk_line_replace_data(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) void pblk_ppa_to_line_put(struct pblk *pblk, struct ppa_addr ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) void pblk_rq_to_line_put(struct pblk *pblk, struct nvm_rq *rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) struct pblk_line *pblk_line_get_data(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) struct pblk_line *pblk_line_get_erase(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) int pblk_line_erase(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) int pblk_line_is_full(struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) void pblk_line_free(struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) void pblk_line_close(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) void pblk_line_close_ws(struct work_struct *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) void pblk_pipeline_stop(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) void __pblk_pipeline_stop(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) void __pblk_pipeline_flush(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) void (*work)(struct work_struct *), gfp_t gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) struct workqueue_struct *wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) void *emeta_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr erase_ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) void pblk_line_put(struct kref *ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) void pblk_line_put_wq(struct kref *ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) void pblk_dealloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) int pblk_calc_secs(struct pblk *pblk, unsigned long secs_avail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) unsigned long secs_to_flush, bool skip_meta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) void pblk_down_rq(struct pblk *pblk, struct ppa_addr ppa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) unsigned long *lun_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) void pblk_down_chunk(struct pblk *pblk, struct ppa_addr ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) void pblk_up_chunk(struct pblk *pblk, struct ppa_addr ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) void pblk_up_rq(struct pblk *pblk, unsigned long *lun_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) int pblk_bio_add_pages(struct pblk *pblk, struct bio *bio, gfp_t flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) int nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) void pblk_bio_free_pages(struct pblk *pblk, struct bio *bio, int off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) int nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) void pblk_map_invalidate(struct pblk *pblk, struct ppa_addr ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) void __pblk_map_invalidate(struct pblk *pblk, struct pblk_line *line,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) u64 paddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) void pblk_update_map(struct pblk *pblk, sector_t lba, struct ppa_addr ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) void pblk_update_map_cache(struct pblk *pblk, sector_t lba,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) struct ppa_addr ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) void pblk_update_map_dev(struct pblk *pblk, sector_t lba,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) struct ppa_addr ppa, struct ppa_addr entry_line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) int pblk_update_map_gc(struct pblk *pblk, sector_t lba, struct ppa_addr ppa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) struct pblk_line *gc_line, u64 paddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) void pblk_lookup_l2p_rand(struct pblk *pblk, struct ppa_addr *ppas,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) u64 *lba_list, int nr_secs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) int pblk_lookup_l2p_seq(struct pblk *pblk, struct ppa_addr *ppas,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) sector_t blba, int nr_secs, bool *from_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) void *pblk_get_meta_for_writes(struct pblk *pblk, struct nvm_rq *rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) void pblk_get_packed_meta(struct pblk *pblk, struct nvm_rq *rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * pblk user I/O write path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) void pblk_write_to_cache(struct pblk *pblk, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) unsigned long flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) int pblk_write_gc_to_cache(struct pblk *pblk, struct pblk_gc_rq *gc_rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) * pblk map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) int pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) unsigned int sentry, unsigned long *lun_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) unsigned int valid_secs, struct ppa_addr *erase_ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) int pblk_map_rq(struct pblk *pblk, struct nvm_rq *rqd, unsigned int sentry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) unsigned long *lun_bitmap, unsigned int valid_secs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) unsigned int off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) * pblk write thread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) int pblk_write_ts(void *data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) void pblk_write_timer_fn(struct timer_list *t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) void pblk_write_should_kick(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) void pblk_write_kick(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * pblk read path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) extern struct bio_set pblk_bio_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) void pblk_submit_read(struct pblk *pblk, struct bio *bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * pblk recovery
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) struct pblk_line *pblk_recov_l2p(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) int pblk_recov_pad(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * pblk gc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) #define PBLK_GC_MAX_READERS 8 /* Max number of outstanding GC reader jobs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) #define PBLK_GC_RQ_QD 128 /* Queue depth for inflight GC requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) #define PBLK_GC_L_QD 4 /* Queue depth for inflight GC lines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) int pblk_gc_init(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) void pblk_gc_exit(struct pblk *pblk, bool graceful);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) void pblk_gc_should_start(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) void pblk_gc_should_stop(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) void pblk_gc_should_kick(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) void pblk_gc_free_full_lines(struct pblk *pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) int *gc_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) int pblk_gc_sysfs_force(struct pblk *pblk, int force);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * pblk rate limiter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) void pblk_rl_init(struct pblk_rl *rl, int budget, int threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) void pblk_rl_free(struct pblk_rl *rl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) void pblk_rl_update_rates(struct pblk_rl *rl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) int pblk_rl_high_thrs(struct pblk_rl *rl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) unsigned long pblk_rl_nr_free_blks(struct pblk_rl *rl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) unsigned long pblk_rl_nr_user_free_blks(struct pblk_rl *rl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) int pblk_rl_user_may_insert(struct pblk_rl *rl, int nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) void pblk_rl_inserted(struct pblk_rl *rl, int nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) void pblk_rl_user_in(struct pblk_rl *rl, int nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) int pblk_rl_gc_may_insert(struct pblk_rl *rl, int nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) void pblk_rl_gc_in(struct pblk_rl *rl, int nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) void pblk_rl_out(struct pblk_rl *rl, int nr_user, int nr_gc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) int pblk_rl_max_io(struct pblk_rl *rl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) void pblk_rl_free_lines_inc(struct pblk_rl *rl, struct pblk_line *line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) void pblk_rl_free_lines_dec(struct pblk_rl *rl, struct pblk_line *line,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) bool used);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) int pblk_rl_is_limit(struct pblk_rl *rl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) void pblk_rl_werr_line_in(struct pblk_rl *rl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) void pblk_rl_werr_line_out(struct pblk_rl *rl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) * pblk sysfs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) int pblk_sysfs_init(struct gendisk *tdisk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) void pblk_sysfs_exit(struct gendisk *tdisk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) static inline struct nvm_rq *nvm_rq_from_c_ctx(void *c_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) return c_ctx - sizeof(struct nvm_rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) static inline void *emeta_to_bb(struct line_emeta *emeta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) return emeta->bb_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) static inline void *emeta_to_wa(struct pblk_line_meta *lm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) struct line_emeta *emeta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) return emeta->bb_bitmap + lm->blk_bitmap_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) static inline void *emeta_to_lbas(struct pblk *pblk, struct line_emeta *emeta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) return ((void *)emeta + pblk->lm.emeta_len[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) static inline void *emeta_to_vsc(struct pblk *pblk, struct line_emeta *emeta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) return (emeta_to_lbas(pblk, emeta) + pblk->lm.emeta_len[2]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) static inline int pblk_line_vsc(struct pblk_line *line)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) return le32_to_cpu(*line->vsc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) static inline int pblk_ppa_to_line_id(struct ppa_addr p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) return p.a.blk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) static inline struct pblk_line *pblk_ppa_to_line(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) struct ppa_addr p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) return &pblk->lines[pblk_ppa_to_line_id(p)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) static inline int pblk_ppa_to_pos(struct nvm_geo *geo, struct ppa_addr p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) return p.a.lun * geo->num_ch + p.a.ch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) static inline struct ppa_addr addr_to_gen_ppa(struct pblk *pblk, u64 paddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) u64 line_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) struct nvm_geo *geo = &dev->geo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) struct ppa_addr ppa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) if (geo->version == NVM_OCSSD_SPEC_12) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) struct nvm_addrf_12 *ppaf = (struct nvm_addrf_12 *)&pblk->addrf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) ppa.ppa = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) ppa.g.blk = line_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) ppa.g.pg = (paddr & ppaf->pg_mask) >> ppaf->pg_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) ppa.g.lun = (paddr & ppaf->lun_mask) >> ppaf->lun_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) ppa.g.ch = (paddr & ppaf->ch_mask) >> ppaf->ch_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) ppa.g.pl = (paddr & ppaf->pln_mask) >> ppaf->pln_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) ppa.g.sec = (paddr & ppaf->sec_mask) >> ppaf->sec_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) struct pblk_addrf *uaddrf = &pblk->uaddrf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) int secs, chnls, luns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) ppa.ppa = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) ppa.m.chk = line_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) paddr = div_u64_rem(paddr, uaddrf->sec_stripe, &secs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) ppa.m.sec = secs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) paddr = div_u64_rem(paddr, uaddrf->ch_stripe, &chnls);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) ppa.m.grp = chnls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) paddr = div_u64_rem(paddr, uaddrf->lun_stripe, &luns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) ppa.m.pu = luns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) ppa.m.sec += uaddrf->sec_stripe * paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) return ppa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) static inline struct nvm_chk_meta *pblk_dev_ppa_to_chunk(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) struct ppa_addr p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) struct nvm_geo *geo = &dev->geo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) struct pblk_line *line = pblk_ppa_to_line(pblk, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) int pos = pblk_ppa_to_pos(geo, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) return &line->chks[pos];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) static inline u64 pblk_dev_ppa_to_chunk_addr(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) struct ppa_addr p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) return dev_to_chunk_addr(dev->parent, &pblk->addrf, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) static inline u64 pblk_dev_ppa_to_line_addr(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) struct ppa_addr p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) struct nvm_geo *geo = &dev->geo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) u64 paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) if (geo->version == NVM_OCSSD_SPEC_12) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) struct nvm_addrf_12 *ppaf = (struct nvm_addrf_12 *)&pblk->addrf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) paddr = (u64)p.g.ch << ppaf->ch_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) paddr |= (u64)p.g.lun << ppaf->lun_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) paddr |= (u64)p.g.pg << ppaf->pg_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) paddr |= (u64)p.g.pl << ppaf->pln_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) paddr |= (u64)p.g.sec << ppaf->sec_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) struct pblk_addrf *uaddrf = &pblk->uaddrf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) u64 secs = p.m.sec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) int sec_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) paddr = (u64)p.m.grp * uaddrf->sec_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) paddr += (u64)p.m.pu * uaddrf->sec_lun_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) secs = div_u64_rem(secs, uaddrf->sec_stripe, &sec_stripe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) paddr += secs * uaddrf->sec_ws_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) paddr += sec_stripe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) return paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) static inline struct ppa_addr pblk_ppa32_to_ppa64(struct pblk *pblk, u32 ppa32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) return nvm_ppa32_to_ppa64(dev->parent, &pblk->addrf, ppa32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) static inline u32 pblk_ppa64_to_ppa32(struct pblk *pblk, struct ppa_addr ppa64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) return nvm_ppa64_to_ppa32(dev->parent, &pblk->addrf, ppa64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) static inline struct ppa_addr pblk_trans_map_get(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) sector_t lba)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) struct ppa_addr ppa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) if (pblk->addrf_len < 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) u32 *map = (u32 *)pblk->trans_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) ppa = pblk_ppa32_to_ppa64(pblk, map[lba]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) struct ppa_addr *map = (struct ppa_addr *)pblk->trans_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) ppa = map[lba];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) return ppa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) static inline void pblk_trans_map_set(struct pblk *pblk, sector_t lba,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) struct ppa_addr ppa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) if (pblk->addrf_len < 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) u32 *map = (u32 *)pblk->trans_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) map[lba] = pblk_ppa64_to_ppa32(pblk, ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) u64 *map = (u64 *)pblk->trans_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) map[lba] = ppa.ppa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) static inline int pblk_ppa_empty(struct ppa_addr ppa_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) return (ppa_addr.ppa == ADDR_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) static inline void pblk_ppa_set_empty(struct ppa_addr *ppa_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) ppa_addr->ppa = ADDR_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) static inline bool pblk_ppa_comp(struct ppa_addr lppa, struct ppa_addr rppa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) return (lppa.ppa == rppa.ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) static inline int pblk_addr_in_cache(struct ppa_addr ppa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) return (ppa.ppa != ADDR_EMPTY && ppa.c.is_cached);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) static inline int pblk_addr_to_cacheline(struct ppa_addr ppa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) return ppa.c.line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) static inline struct ppa_addr pblk_cacheline_to_addr(int addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) struct ppa_addr p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) p.c.line = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) p.c.is_cached = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) return p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) static inline u32 pblk_calc_meta_header_crc(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) struct line_header *header)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) u32 crc = ~(u32)0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) crc = crc32_le(crc, (unsigned char *)header + sizeof(crc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) sizeof(struct line_header) - sizeof(crc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) return crc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) static inline u32 pblk_calc_smeta_crc(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) struct line_smeta *smeta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) struct pblk_line_meta *lm = &pblk->lm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) u32 crc = ~(u32)0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) crc = crc32_le(crc, (unsigned char *)smeta +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) sizeof(struct line_header) + sizeof(crc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) lm->smeta_len -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) sizeof(struct line_header) - sizeof(crc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) return crc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) static inline u32 pblk_calc_emeta_crc(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) struct line_emeta *emeta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) struct pblk_line_meta *lm = &pblk->lm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) u32 crc = ~(u32)0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) crc = crc32_le(crc, (unsigned char *)emeta +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) sizeof(struct line_header) + sizeof(crc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) lm->emeta_len[0] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) sizeof(struct line_header) - sizeof(crc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) return crc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) static inline int pblk_io_aligned(struct pblk *pblk, int nr_secs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) return !(nr_secs % pblk->min_write_pgs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) static inline void print_ppa(struct pblk *pblk, struct ppa_addr *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) char *msg, int error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) struct nvm_geo *geo = &pblk->dev->geo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) if (p->c.is_cached) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) pblk_err(pblk, "ppa: (%s: %x) cache line: %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) msg, error, (u64)p->c.line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) } else if (geo->version == NVM_OCSSD_SPEC_12) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) pblk_err(pblk, "ppa: (%s: %x):ch:%d,lun:%d,blk:%d,pg:%d,pl:%d,sec:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) msg, error,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) p->g.ch, p->g.lun, p->g.blk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) p->g.pg, p->g.pl, p->g.sec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) pblk_err(pblk, "ppa: (%s: %x):ch:%d,lun:%d,chk:%d,sec:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) msg, error,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) p->m.grp, p->m.pu, p->m.chk, p->m.sec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) static inline void pblk_print_failed_rqd(struct pblk *pblk, struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) int error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) int bit = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) if (rqd->nr_ppas == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) print_ppa(pblk, &rqd->ppa_addr, "rqd", error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) while ((bit = find_next_bit((void *)&rqd->ppa_status, rqd->nr_ppas,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) bit + 1)) < rqd->nr_ppas) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) print_ppa(pblk, &rqd->ppa_list[bit], "rqd", error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) pblk_err(pblk, "error:%d, ppa_status:%llx\n", error, rqd->ppa_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) static inline int pblk_boundary_ppa_checks(struct nvm_tgt_dev *tgt_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) struct ppa_addr *ppas, int nr_ppas)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) struct nvm_geo *geo = &tgt_dev->geo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) struct ppa_addr *ppa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) for (i = 0; i < nr_ppas; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) ppa = &ppas[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) if (geo->version == NVM_OCSSD_SPEC_12) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) if (!ppa->c.is_cached &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) ppa->g.ch < geo->num_ch &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) ppa->g.lun < geo->num_lun &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) ppa->g.pl < geo->num_pln &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) ppa->g.blk < geo->num_chk &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) ppa->g.pg < geo->num_pg &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) ppa->g.sec < geo->ws_min)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) if (!ppa->c.is_cached &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) ppa->m.grp < geo->num_ch &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) ppa->m.pu < geo->num_lun &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) ppa->m.chk < geo->num_chk &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) ppa->m.sec < geo->clba)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) print_ppa(tgt_dev->q->queuedata, ppa, "boundary", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) static inline int pblk_check_io(struct pblk *pblk, struct nvm_rq *rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) if (pblk_boundary_ppa_checks(dev, ppa_list, rqd->nr_ppas)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) if (rqd->opcode == NVM_OP_PWRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) struct pblk_line *line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) for (i = 0; i < rqd->nr_ppas; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) line = pblk_ppa_to_line(pblk, ppa_list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) spin_lock(&line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) if (line->state != PBLK_LINESTATE_OPEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) pblk_err(pblk, "bad ppa: line:%d,state:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) line->id, line->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) spin_unlock(&line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) spin_unlock(&line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) static inline int pblk_boundary_paddr_checks(struct pblk *pblk, u64 paddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) struct pblk_line_meta *lm = &pblk->lm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) if (paddr > lm->sec_per_line)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) static inline unsigned int pblk_get_bi_idx(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) return bio->bi_iter.bi_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) static inline sector_t pblk_get_lba(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) return bio->bi_iter.bi_sector / NR_PHY_IN_LOG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) static inline unsigned int pblk_get_secs(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) return bio->bi_iter.bi_size / PBLK_EXPOSED_PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) static inline char *pblk_disk_name(struct pblk *pblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) struct gendisk *disk = pblk->disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) return disk->disk_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) static inline unsigned int pblk_get_min_chks(struct pblk *pblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) struct pblk_line_meta *lm = &pblk->lm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) /* In a worst-case scenario every line will have OP invalid sectors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) * We will then need a minimum of 1/OP lines to free up a single line
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) return DIV_ROUND_UP(100, pblk->op) * lm->blk_per_line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) static inline struct pblk_sec_meta *pblk_get_meta(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) void *meta, int index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) return meta +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) max_t(int, sizeof(struct pblk_sec_meta), pblk->oob_meta_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) * index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) static inline int pblk_dma_meta_size(struct pblk *pblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) return max_t(int, sizeof(struct pblk_sec_meta), pblk->oob_meta_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) * NVM_MAX_VLBA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) static inline int pblk_is_oob_meta_supported(struct pblk *pblk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) return pblk->oob_meta_size >= sizeof(struct pblk_sec_meta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) #endif /* PBLK_H_ */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags