^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2016 CNEX Labs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Initial release: Javier Gonzalez <javier@cnexlabs.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Matias Bjorling <matias@cnexlabs.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * This program is free software; you can redistribute it and/or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * modify it under the terms of the GNU General Public License version
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * 2 as published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * pblk-map.c - pblk's lba-ppa mapping strategy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "pblk.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static int pblk_map_page_data(struct pblk *pblk, unsigned int sentry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) struct ppa_addr *ppa_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) unsigned long *lun_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) void *meta_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) unsigned int valid_secs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct pblk_line *line = pblk_line_get_data(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) struct pblk_emeta *emeta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) struct pblk_w_ctx *w_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) __le64 *lba_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) u64 paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) int nr_secs = pblk->min_write_pgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) if (!line)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (pblk_line_is_full(line)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct pblk_line *prev_line = line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* If we cannot allocate a new line, make sure to store metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * on current line and then fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) line = pblk_line_replace_data(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) pblk_line_close_meta(pblk, prev_line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) if (!line) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) pblk_pipeline_stop(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) emeta = line->emeta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) lba_list = emeta_to_lbas(pblk, emeta->buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) paddr = pblk_alloc_page(pblk, line, nr_secs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) for (i = 0; i < nr_secs; i++, paddr++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) __le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) /* ppa to be sent to the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /* Write context for target bio completion on write buffer. Note
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * that the write buffer is protected by the sync backpointer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * and a single writer thread have access to each specific entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * at a time. Thus, it is safe to modify the context for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * entry we are setting up for submission without taking any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * lock or memory barrier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if (i < valid_secs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) kref_get(&line->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) atomic_inc(&line->sec_to_update);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) w_ctx = pblk_rb_w_ctx(&pblk->rwb, sentry + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) w_ctx->ppa = ppa_list[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) meta->lba = cpu_to_le64(w_ctx->lba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) lba_list[paddr] = cpu_to_le64(w_ctx->lba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) if (lba_list[paddr] != addr_empty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) line->nr_valid_lbas++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) atomic64_inc(&pblk->pad_wa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) lba_list[paddr] = addr_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) meta->lba = addr_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) __pblk_map_invalidate(pblk, line, paddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) pblk_down_rq(pblk, ppa_list[0], lun_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) int pblk_map_rq(struct pblk *pblk, struct nvm_rq *rqd, unsigned int sentry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) unsigned long *lun_bitmap, unsigned int valid_secs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) unsigned int off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) void *meta_list = pblk_get_meta_for_writes(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) void *meta_buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) unsigned int map_secs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) int min = pblk->min_write_pgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) for (i = off; i < rqd->nr_ppas; i += min) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) map_secs = (i + min > valid_secs) ? (valid_secs % min) : min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) meta_buffer = pblk_get_meta(pblk, meta_list, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) ret = pblk_map_page_data(pblk, sentry + i, &ppa_list[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) lun_bitmap, meta_buffer, map_secs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) /* only if erase_ppa is set, acquire erase semaphore */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) int pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) unsigned int sentry, unsigned long *lun_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) unsigned int valid_secs, struct ppa_addr *erase_ppa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) struct nvm_geo *geo = &dev->geo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) struct pblk_line_meta *lm = &pblk->lm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) void *meta_list = pblk_get_meta_for_writes(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) void *meta_buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct pblk_line *e_line, *d_line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) unsigned int map_secs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) int min = pblk->min_write_pgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) int i, erase_lun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) for (i = 0; i < rqd->nr_ppas; i += min) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) map_secs = (i + min > valid_secs) ? (valid_secs % min) : min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) meta_buffer = pblk_get_meta(pblk, meta_list, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) ret = pblk_map_page_data(pblk, sentry + i, &ppa_list[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) lun_bitmap, meta_buffer, map_secs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) erase_lun = pblk_ppa_to_pos(geo, ppa_list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /* line can change after page map. We might also be writing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * last line.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) e_line = pblk_line_get_erase(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (!e_line)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return pblk_map_rq(pblk, rqd, sentry, lun_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) valid_secs, i + min);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) spin_lock(&e_line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (!test_bit(erase_lun, e_line->erase_bitmap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) set_bit(erase_lun, e_line->erase_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) atomic_dec(&e_line->left_eblks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) *erase_ppa = ppa_list[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) erase_ppa->a.blk = e_line->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) erase_ppa->a.reserved = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) spin_unlock(&e_line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /* Avoid evaluating e_line->left_eblks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) return pblk_map_rq(pblk, rqd, sentry, lun_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) valid_secs, i + min);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) spin_unlock(&e_line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) d_line = pblk_line_get_data(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /* line can change after page map. We might also be writing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * last line.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) e_line = pblk_line_get_erase(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (!e_line)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /* Erase blocks that are bad in this line but might not be in next */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (unlikely(pblk_ppa_empty(*erase_ppa)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) bitmap_weight(d_line->blk_bitmap, lm->blk_per_line)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) int bit = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) bit = find_next_bit(d_line->blk_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) lm->blk_per_line, bit + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (bit >= lm->blk_per_line)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) spin_lock(&e_line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (test_bit(bit, e_line->erase_bitmap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) spin_unlock(&e_line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) spin_unlock(&e_line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) set_bit(bit, e_line->erase_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) atomic_dec(&e_line->left_eblks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) *erase_ppa = pblk->luns[bit].bppa; /* set ch and lun */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) erase_ppa->a.blk = e_line->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags