^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2016 CNEX Labs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Initial release: Javier Gonzalez <javier@cnexlabs.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Matias Bjorling <matias@cnexlabs.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * This program is free software; you can redistribute it and/or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * modify it under the terms of the GNU General Public License version
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * 2 as published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * pblk-write.c - pblk's write path from write buffer to media
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "pblk.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "pblk-trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) struct pblk_c_ctx *c_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) struct bio *original_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct pblk_rb *rwb = &pblk->rwb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) unsigned long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) for (i = 0; i < c_ctx->nr_valid; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct pblk_w_ctx *w_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) int pos = c_ctx->sentry + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) int flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) w_ctx = pblk_rb_w_ctx(rwb, pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) flags = READ_ONCE(w_ctx->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) if (flags & PBLK_FLUSH_ENTRY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) flags &= ~PBLK_FLUSH_ENTRY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /* Release flags on context. Protect from writes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) smp_store_release(&w_ctx->flags, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) atomic_dec(&rwb->inflight_flush_point);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) while ((original_bio = bio_list_pop(&w_ctx->bios)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) bio_endio(original_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) if (c_ctx->nr_padded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) c_ctx->nr_padded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) atomic_long_add(rqd->nr_ppas, &pblk->sync_writes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) ret = pblk_rb_sync_advance(&pblk->rwb, c_ctx->nr_valid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) bio_put(rqd->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) pblk_free_rqd(pblk, rqd, PBLK_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static unsigned long pblk_end_queued_w_bio(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct pblk_c_ctx *c_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) list_del(&c_ctx->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return pblk_end_w_bio(pblk, rqd, c_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) struct pblk_c_ctx *c_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct pblk_c_ctx *c, *r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) unsigned long pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) atomic_long_sub(c_ctx->nr_valid, &pblk->inflight_writes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) pblk_up_rq(pblk, c_ctx->lun_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) pos = pblk_rb_sync_init(&pblk->rwb, &flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (pos == c_ctx->sentry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) pos = pblk_end_w_bio(pblk, rqd, c_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) list_for_each_entry_safe(c, r, &pblk->compl_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) rqd = nvm_rq_from_c_ctx(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (c->sentry == pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) pos = pblk_end_queued_w_bio(pblk, rqd, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) WARN_ON(nvm_rq_from_c_ctx(c_ctx) != rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) list_add_tail(&c_ctx->list, &pblk->compl_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) pblk_rb_sync_end(&pblk->rwb, &flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* Map remaining sectors in chunk, starting from ppa */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static void pblk_map_remaining(struct pblk *pblk, struct ppa_addr *ppa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) int rqd_ppas)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct pblk_line *line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) struct ppa_addr map_ppa = *ppa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) __le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) __le64 *lba_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) u64 paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) int done = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) int n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) line = pblk_ppa_to_line(pblk, *ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) lba_list = emeta_to_lbas(pblk, line->emeta->buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) spin_lock(&line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) while (!done) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) paddr = pblk_dev_ppa_to_line_addr(pblk, map_ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) if (!test_and_set_bit(paddr, line->map_bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) line->left_msecs--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) if (n < rqd_ppas && lba_list[paddr] != addr_empty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) line->nr_valid_lbas--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) lba_list[paddr] = addr_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (!test_and_set_bit(paddr, line->invalid_bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) le32_add_cpu(line->vsc, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) done = nvm_next_ppa_in_chk(pblk->dev, &map_ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) n++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) line->w_err_gc->has_write_err = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) spin_unlock(&line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) static void pblk_prepare_resubmit(struct pblk *pblk, unsigned int sentry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) unsigned int nr_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) struct pblk_rb *rb = &pblk->rwb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) struct pblk_rb_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) struct pblk_line *line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct pblk_w_ctx *w_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) struct ppa_addr ppa_l2p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) int flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) spin_lock(&pblk->trans_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) for (i = 0; i < nr_entries; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) entry = &rb->entries[pblk_rb_ptr_wrap(rb, sentry, i)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) w_ctx = &entry->w_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) /* Check if the lba has been overwritten */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (w_ctx->lba != ADDR_EMPTY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) ppa_l2p = pblk_trans_map_get(pblk, w_ctx->lba);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (!pblk_ppa_comp(ppa_l2p, entry->cacheline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) w_ctx->lba = ADDR_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) /* Mark up the entry as submittable again */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) flags = READ_ONCE(w_ctx->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) flags |= PBLK_WRITTEN_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* Release flags on write context. Protect from writes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) smp_store_release(&w_ctx->flags, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) /* Decrease the reference count to the line as we will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * re-map these entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) line = pblk_ppa_to_line(pblk, w_ctx->ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) atomic_dec(&line->sec_to_update);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) kref_put(&line->ref, pblk_line_put);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) spin_unlock(&pblk->trans_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) static void pblk_queue_resubmit(struct pblk *pblk, struct pblk_c_ctx *c_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) struct pblk_c_ctx *r_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) r_ctx = kzalloc(sizeof(struct pblk_c_ctx), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (!r_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) r_ctx->lun_bitmap = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) r_ctx->sentry = c_ctx->sentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) r_ctx->nr_valid = c_ctx->nr_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) r_ctx->nr_padded = c_ctx->nr_padded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) spin_lock(&pblk->resubmit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) list_add_tail(&r_ctx->list, &pblk->resubmit_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) spin_unlock(&pblk->resubmit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) atomic_long_add(c_ctx->nr_valid, &pblk->recov_writes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static void pblk_submit_rec(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct pblk_rec_ctx *recovery =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) container_of(work, struct pblk_rec_ctx, ws_rec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) struct pblk *pblk = recovery->pblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct nvm_rq *rqd = recovery->rqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) pblk_log_write_err(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) pblk_map_remaining(pblk, ppa_list, rqd->nr_ppas);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) pblk_queue_resubmit(pblk, c_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) pblk_up_rq(pblk, c_ctx->lun_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (c_ctx->nr_padded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) c_ctx->nr_padded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) bio_put(rqd->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) pblk_free_rqd(pblk, rqd, PBLK_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) mempool_free(recovery, &pblk->rec_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) atomic_dec(&pblk->inflight_io);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) pblk_write_kick(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) struct pblk_rec_ctx *recovery;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) recovery = mempool_alloc(&pblk->rec_pool, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (!recovery) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) pblk_err(pblk, "could not allocate recovery work\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) recovery->pblk = pblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) recovery->rqd = rqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) INIT_WORK(&recovery->ws_rec, pblk_submit_rec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) queue_work(pblk->close_wq, &recovery->ws_rec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) static void pblk_end_io_write(struct nvm_rq *rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct pblk *pblk = rqd->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (rqd->error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) pblk_end_w_fail(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (trace_pblk_chunk_state_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) pblk_check_chunk_state_update(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) WARN_ONCE(rqd->bio->bi_status, "pblk: corrupted write error\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) pblk_complete_write(pblk, rqd, c_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) atomic_dec(&pblk->inflight_io);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) static void pblk_end_io_write_meta(struct nvm_rq *rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct pblk *pblk = rqd->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct pblk_g_ctx *m_ctx = nvm_rq_to_pdu(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) struct pblk_line *line = m_ctx->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) struct pblk_emeta *emeta = line->emeta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) int sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) pblk_up_chunk(pblk, ppa_list[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) if (rqd->error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) pblk_log_write_err(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) pblk_err(pblk, "metadata I/O failed. Line %d\n", line->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) line->w_err_gc->has_write_err = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (trace_pblk_chunk_state_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) pblk_check_chunk_state_update(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) sync = atomic_add_return(rqd->nr_ppas, &emeta->sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) if (sync == emeta->nr_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) pblk_gen_run_ws(pblk, line, NULL, pblk_line_close_ws,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) GFP_ATOMIC, pblk->close_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) atomic_dec(&pblk->inflight_io);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) unsigned int nr_secs, nvm_end_io_fn(*end_io))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) /* Setup write request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) rqd->opcode = NVM_OP_PWRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) rqd->nr_ppas = nr_secs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) rqd->is_seq = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) rqd->private = pblk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) rqd->end_io = end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) return pblk_alloc_rqd_meta(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct ppa_addr *erase_ppa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct pblk_line_meta *lm = &pblk->lm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) struct pblk_line *e_line = pblk_line_get_erase(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) unsigned int valid = c_ctx->nr_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) unsigned int padded = c_ctx->nr_padded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) unsigned int nr_secs = valid + padded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) unsigned long *lun_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) lun_bitmap = kzalloc(lm->lun_bitmap_len, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (!lun_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) c_ctx->lun_bitmap = lun_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) ret = pblk_alloc_w_rq(pblk, rqd, nr_secs, pblk_end_io_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) kfree(lun_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (likely(!e_line || !atomic_read(&e_line->left_eblks)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) ret = pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) valid, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) ret = pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) valid, erase_ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static int pblk_calc_secs_to_sync(struct pblk *pblk, unsigned int secs_avail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) unsigned int secs_to_flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) int secs_to_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) secs_to_sync = pblk_calc_secs(pblk, secs_avail, secs_to_flush, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if ((!secs_to_sync && secs_to_flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) || (secs_to_sync < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) || (secs_to_sync > secs_avail && !secs_to_flush)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) pblk_err(pblk, "bad sector calculation (a:%d,s:%d,f:%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) secs_avail, secs_to_sync, secs_to_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) return secs_to_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct nvm_geo *geo = &dev->geo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) struct pblk_line_mgmt *l_mg = &pblk->l_mg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) struct pblk_line_meta *lm = &pblk->lm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct pblk_emeta *emeta = meta_line->emeta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) struct ppa_addr *ppa_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) struct pblk_g_ctx *m_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) struct nvm_rq *rqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) void *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) u64 paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) int rq_ppas = pblk->min_write_pgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) int id = meta_line->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) int rq_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) m_ctx = nvm_rq_to_pdu(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) m_ctx->private = meta_line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) rq_len = rq_ppas * geo->csecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) data = ((void *)emeta->buf) + emeta->mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) ret = pblk_alloc_w_rq(pblk, rqd, rq_ppas, pblk_end_io_write_meta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) goto fail_free_rqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) ppa_list = nvm_rq_to_ppa_list(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) for (i = 0; i < rqd->nr_ppas; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) spin_lock(&meta_line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) paddr = __pblk_alloc_page(pblk, meta_line, rq_ppas);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) spin_unlock(&meta_line->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) for (j = 0; j < rq_ppas; j++, i++, paddr++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) spin_lock(&l_mg->close_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) emeta->mem += rq_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (emeta->mem >= lm->emeta_len[0])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) list_del(&meta_line->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) spin_unlock(&l_mg->close_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) pblk_down_chunk(pblk, ppa_list[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) ret = pblk_submit_io(pblk, rqd, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) pblk_err(pblk, "emeta I/O submission failed: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) goto fail_rollback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) return NVM_IO_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) fail_rollback:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) pblk_up_chunk(pblk, ppa_list[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) spin_lock(&l_mg->close_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) pblk_dealloc_page(pblk, meta_line, rq_ppas);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) list_add(&meta_line->list, &meta_line->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) spin_unlock(&l_mg->close_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) fail_free_rqd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) static inline bool pblk_valid_meta_ppa(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) struct pblk_line *meta_line,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) struct nvm_rq *data_rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) struct nvm_geo *geo = &dev->geo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) struct pblk_c_ctx *data_c_ctx = nvm_rq_to_pdu(data_rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) struct pblk_line *data_line = pblk_line_get_data(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) struct ppa_addr ppa, ppa_opt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) u64 paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) int pos_opt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) /* Schedule a metadata I/O that is half the distance from the data I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * with regards to the number of LUNs forming the pblk instance. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * balances LUN conflicts across every I/O.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * When the LUN configuration changes (e.g., due to GC), this distance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * can align, which would result on metadata and data I/Os colliding. In
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * this case, modify the distance to not be optimal, but move the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * optimal in the right direction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) paddr = pblk_lookup_page(pblk, meta_line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) ppa = addr_to_gen_ppa(pblk, paddr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) ppa_opt = addr_to_gen_ppa(pblk, paddr + data_line->meta_distance, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) pos_opt = pblk_ppa_to_pos(geo, ppa_opt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (test_bit(pos_opt, data_c_ctx->lun_bitmap) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) test_bit(pos_opt, data_line->blk_bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) if (unlikely(pblk_ppa_comp(ppa_opt, ppa)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) data_line->meta_distance--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) static struct pblk_line *pblk_should_submit_meta_io(struct pblk *pblk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) struct nvm_rq *data_rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) struct pblk_line_meta *lm = &pblk->lm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) struct pblk_line_mgmt *l_mg = &pblk->l_mg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) struct pblk_line *meta_line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) spin_lock(&l_mg->close_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (list_empty(&l_mg->emeta_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) spin_unlock(&l_mg->close_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) meta_line = list_first_entry(&l_mg->emeta_list, struct pblk_line, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) if (meta_line->emeta->mem >= lm->emeta_len[0]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) spin_unlock(&l_mg->close_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) spin_unlock(&l_mg->close_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (!pblk_valid_meta_ppa(pblk, meta_line, data_rqd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return meta_line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) struct ppa_addr erase_ppa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) struct pblk_line *meta_line;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) pblk_ppa_set_empty(&erase_ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) /* Assign lbas to ppas and populate request structure */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) err = pblk_setup_w_rq(pblk, rqd, &erase_ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) pblk_err(pblk, "could not setup write request: %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return NVM_IO_ERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) meta_line = pblk_should_submit_meta_io(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) /* Submit data write for current data line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) err = pblk_submit_io(pblk, rqd, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) pblk_err(pblk, "data I/O submission failed: %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) return NVM_IO_ERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) if (!pblk_ppa_empty(erase_ppa)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /* Submit erase for next data line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) if (pblk_blk_erase_async(pblk, erase_ppa)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) struct pblk_line *e_line = pblk_line_get_erase(pblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) struct nvm_tgt_dev *dev = pblk->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) struct nvm_geo *geo = &dev->geo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) int bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) atomic_inc(&e_line->left_eblks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) bit = pblk_ppa_to_pos(geo, erase_ppa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) WARN_ON(!test_and_clear_bit(bit, e_line->erase_bitmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (meta_line) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) /* Submit metadata write for previous data line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) err = pblk_submit_meta_io(pblk, meta_line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) pblk_err(pblk, "metadata I/O submission failed: %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) return NVM_IO_ERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) return NVM_IO_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) static void pblk_free_write_rqd(struct pblk *pblk, struct nvm_rq *rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) struct bio *bio = rqd->bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (c_ctx->nr_padded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) pblk_bio_free_pages(pblk, bio, c_ctx->nr_valid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) c_ctx->nr_padded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) static int pblk_submit_write(struct pblk *pblk, int *secs_left)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) struct nvm_rq *rqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) unsigned int secs_avail, secs_to_sync, secs_to_com;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) unsigned int secs_to_flush, packed_meta_pgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) unsigned long pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) unsigned int resubmit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) *secs_left = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) spin_lock(&pblk->resubmit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) resubmit = !list_empty(&pblk->resubmit_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) spin_unlock(&pblk->resubmit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) /* Resubmit failed writes first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if (resubmit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) struct pblk_c_ctx *r_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) spin_lock(&pblk->resubmit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) r_ctx = list_first_entry(&pblk->resubmit_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) struct pblk_c_ctx, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) list_del(&r_ctx->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) spin_unlock(&pblk->resubmit_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) secs_avail = r_ctx->nr_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) pos = r_ctx->sentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) pblk_prepare_resubmit(pblk, pos, secs_avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) secs_avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) kfree(r_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) /* If there are no sectors in the cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) * flushes (bios without data) will be cleared on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * the cache threads
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) secs_avail = pblk_rb_read_count(&pblk->rwb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (!secs_avail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) secs_to_flush = pblk_rb_flush_point_count(&pblk->rwb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (!secs_to_flush && secs_avail < pblk->min_write_pgs_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) secs_to_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (secs_to_sync > pblk->max_write_pgs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) pblk_err(pblk, "bad buffer sync calculation\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) secs_to_com = (secs_to_sync > secs_avail) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) secs_avail : secs_to_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) pos = pblk_rb_read_commit(&pblk->rwb, secs_to_com);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) packed_meta_pgs = (pblk->min_write_pgs - pblk->min_write_pgs_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) bio = bio_alloc(GFP_KERNEL, secs_to_sync + packed_meta_pgs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) bio->bi_iter.bi_sector = 0; /* internal bio */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) rqd = pblk_alloc_rqd(pblk, PBLK_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) rqd->bio = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) if (pblk_rb_read_to_bio(&pblk->rwb, rqd, pos, secs_to_sync,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) secs_avail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) pblk_err(pblk, "corrupted write bio\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) goto fail_put_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (pblk_submit_io_set(pblk, rqd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) goto fail_free_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) #ifdef CONFIG_NVM_PBLK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) atomic_long_add(secs_to_sync, &pblk->sub_writes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) *secs_left = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) fail_free_bio:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) pblk_free_write_rqd(pblk, rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) fail_put_bio:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) pblk_free_rqd(pblk, rqd, PBLK_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) int pblk_write_ts(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) struct pblk *pblk = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) int secs_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) int write_failure = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) while (!kthread_should_stop()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) if (!write_failure) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) write_failure = pblk_submit_write(pblk, &secs_left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) if (secs_left)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) set_current_state(TASK_INTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) io_schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags