^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) drbd_req.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/drbd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "drbd_int.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "drbd_req.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static bool drbd_may_do_local_read(struct drbd_device *device, sector_t sector, int size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio *bio_src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct drbd_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) req = mempool_alloc(&drbd_request_mempool, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) if (!req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) memset(req, 0, sizeof(*req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) drbd_req_make_private_bio(req, bio_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) | (bio_op(bio_src) == REQ_OP_WRITE_SAME ? RQ_WSAME : 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) | (bio_op(bio_src) == REQ_OP_WRITE_ZEROES ? RQ_ZEROES : 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) | (bio_op(bio_src) == REQ_OP_DISCARD ? RQ_UNMAP : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) req->device = device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) req->master_bio = bio_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) req->epoch = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) drbd_clear_interval(&req->i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) req->i.sector = bio_src->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) req->i.size = bio_src->bi_iter.bi_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) req->i.local = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) req->i.waiting = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) INIT_LIST_HEAD(&req->tl_requests);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) INIT_LIST_HEAD(&req->w.list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) INIT_LIST_HEAD(&req->req_pending_master_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) INIT_LIST_HEAD(&req->req_pending_local);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /* one reference to be put by __drbd_make_request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) atomic_set(&req->completion_ref, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /* one kref as long as completion_ref > 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) kref_init(&req->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) return req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static void drbd_remove_request_interval(struct rb_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct drbd_device *device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct drbd_interval *i = &req->i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) drbd_remove_interval(root, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* Wake up any processes waiting for this request to complete. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) if (i->waiting)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) wake_up(&device->misc_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) void drbd_req_destroy(struct kref *kref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct drbd_request *req = container_of(kref, struct drbd_request, kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct drbd_device *device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) const unsigned s = req->rq_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if ((req->master_bio && !(s & RQ_POSTPONED)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) atomic_read(&req->completion_ref) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) (s & RQ_LOCAL_PENDING) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) ((s & RQ_NET_MASK) && !(s & RQ_NET_DONE))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) drbd_err(device, "drbd_req_destroy: Logic BUG rq_state = 0x%x, completion_ref = %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) s, atomic_read(&req->completion_ref));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /* If called from mod_rq_state (expected normal case) or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * drbd_send_and_submit (the less likely normal path), this holds the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * req_lock, and req->tl_requests will typicaly be on ->transfer_log,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * though it may be still empty (never added to the transfer log).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * If called from do_retry(), we do NOT hold the req_lock, but we are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * still allowed to unconditionally list_del(&req->tl_requests),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * because it will be on a local on-stack list only. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) list_del_init(&req->tl_requests);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /* finally remove the request from the conflict detection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * respective block_id verification interval tree. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (!drbd_interval_empty(&req->i)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct rb_root *root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (s & RQ_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) root = &device->write_requests;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) root = &device->read_requests;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) drbd_remove_request_interval(root, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) } else if (s & (RQ_NET_MASK & ~RQ_NET_DONE) && req->i.size != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) drbd_err(device, "drbd_req_destroy: Logic BUG: interval empty, but: rq_state=0x%x, sect=%llu, size=%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) s, (unsigned long long)req->i.sector, req->i.size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /* if it was a write, we may have to set the corresponding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * bit(s) out-of-sync first. If it had a local part, we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * release the reference to the activity log. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (s & RQ_WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /* Set out-of-sync unless both OK flags are set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * (local only or remote failed).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * Other places where we set out-of-sync:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * READ with local io-error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) /* There is a special case:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * we may notice late that IO was suspended,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * and postpone, or schedule for retry, a write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * before it even was submitted or sent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * In that case we do not want to touch the bitmap at all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) if ((s & (RQ_POSTPONED|RQ_LOCAL_MASK|RQ_NET_MASK)) != RQ_POSTPONED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (!(s & RQ_NET_OK) || !(s & RQ_LOCAL_OK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) drbd_set_out_of_sync(device, req->i.sector, req->i.size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if ((s & RQ_NET_OK) && (s & RQ_LOCAL_OK) && (s & RQ_NET_SIS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) drbd_set_in_sync(device, req->i.sector, req->i.size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) /* one might be tempted to move the drbd_al_complete_io
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * to the local io completion callback drbd_request_endio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * but, if this was a mirror write, we may only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * drbd_al_complete_io after this is RQ_NET_DONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * otherwise the extent could be dropped from the al
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * before it has actually been written on the peer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * if we crash before our peer knows about the request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * but after the extent has been dropped from the al,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * we would forget to resync the corresponding extent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (s & RQ_IN_ACT_LOG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (get_ldev_if_state(device, D_FAILED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) drbd_al_complete_io(device, &req->i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) put_ldev(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) } else if (__ratelimit(&drbd_ratelimit_state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) drbd_warn(device, "Should have called drbd_al_complete_io(, %llu, %u), "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) "but my Disk seems to have failed :(\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) (unsigned long long) req->i.sector, req->i.size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) mempool_free(req, &drbd_request_mempool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) static void wake_all_senders(struct drbd_connection *connection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) wake_up(&connection->sender_work.q_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /* must hold resource->req_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) void start_new_tl_epoch(struct drbd_connection *connection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /* no point closing an epoch, if it is empty, anyways. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (connection->current_tle_writes == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) connection->current_tle_writes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) atomic_inc(&connection->current_tle_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) wake_all_senders(connection);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) void complete_master_bio(struct drbd_device *device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) struct bio_and_error *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (unlikely(m->error))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) m->bio->bi_status = errno_to_blk_status(m->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) bio_endio(m->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) dec_ap_bio(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /* Helper for __req_mod().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * Set m->bio to the master bio, if it is fit to be completed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * or leave it alone (it is initialized to NULL in __req_mod),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * if it has already been completed, or cannot be completed yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * If m->bio is set, the error status to be returned is placed in m->error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) static
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) const unsigned s = req->rq_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) struct drbd_device *device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) int error, ok;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) /* we must not complete the master bio, while it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * still being processed by _drbd_send_zc_bio (drbd_send_dblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * not yet acknowledged by the peer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * not yet completed by the local io subsystem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * these flags may get cleared in any order by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * the worker,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * the receiver,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * the bio_endio completion callbacks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if ((s & RQ_LOCAL_PENDING && !(s & RQ_LOCAL_ABORTED)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) (s & RQ_NET_QUEUED) || (s & RQ_NET_PENDING) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) (s & RQ_COMPLETION_SUSP)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) drbd_err(device, "drbd_req_complete: Logic BUG rq_state = 0x%x\n", s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) if (!req->master_bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) drbd_err(device, "drbd_req_complete: Logic BUG, master_bio == NULL!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * figure out whether to report success or failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * report success when at least one of the operations succeeded.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * or, to put the other way,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * only report failure, when both operations failed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * what to do about the failures is handled elsewhere.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * what we need to do here is just: complete the master_bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * local completion error, if any, has been stored as ERR_PTR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * in private_bio within drbd_request_endio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) ok = (s & RQ_LOCAL_OK) || (s & RQ_NET_OK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) error = PTR_ERR(req->private_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) /* Before we can signal completion to the upper layers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * we may need to close the current transfer log epoch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * We are within the request lock, so we can simply compare
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * the request epoch number with the current transfer log
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * epoch number. If they match, increase the current_tle_nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * and reset the transfer log epoch write_cnt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if (op_is_write(bio_op(req->master_bio)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) req->epoch == atomic_read(&first_peer_device(device)->connection->current_tle_nr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) start_new_tl_epoch(first_peer_device(device)->connection);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) /* Update disk stats */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) bio_end_io_acct(req->master_bio, req->start_jif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) /* If READ failed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * have it be pushed back to the retry work queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * so it will re-enter __drbd_make_request(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * and be re-assigned to a suitable local or remote path,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * or failed if we do not have access to good data anymore.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * Unless it was failed early by __drbd_make_request(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * because no path was available, in which case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * it was not even added to the transfer_log.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * read-ahead may fail, and will not be retried.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * WRITE should have used all available paths already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (!ok &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) bio_op(req->master_bio) == REQ_OP_READ &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) !(req->master_bio->bi_opf & REQ_RAHEAD) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) !list_empty(&req->tl_requests))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) req->rq_state |= RQ_POSTPONED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (!(req->rq_state & RQ_POSTPONED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) m->error = ok ? 0 : (error ?: -EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) m->bio = req->master_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) req->master_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) /* We leave it in the tree, to be able to verify later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * write-acks in protocol != C during resync.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * But we mark it as "complete", so it won't be counted as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * conflict in a multi-primary setup. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) req->i.completed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (req->i.waiting)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) wake_up(&device->misc_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /* Either we are about to complete to upper layers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * or we will restart this request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * In either case, the request object will be destroyed soon,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * so better remove it from all lists. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) list_del_init(&req->req_pending_master_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) /* still holds resource->req_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static void drbd_req_put_completion_ref(struct drbd_request *req, struct bio_and_error *m, int put)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct drbd_device *device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) D_ASSERT(device, m || (req->rq_state & RQ_POSTPONED));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (!put)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (!atomic_sub_and_test(put, &req->completion_ref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) drbd_req_complete(req, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /* local completion may still come in later,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * we need to keep the req object around. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (req->rq_state & RQ_LOCAL_ABORTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (req->rq_state & RQ_POSTPONED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /* don't destroy the req object just yet,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * but queue it for retry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) drbd_restart_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) kref_put(&req->kref, drbd_req_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static void set_if_null_req_next(struct drbd_peer_device *peer_device, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (!connection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (connection->req_next == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) connection->req_next = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) static void advance_conn_req_next(struct drbd_peer_device *peer_device, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (!connection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (connection->req_next != req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) const unsigned s = req->rq_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (s & RQ_NET_QUEUED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (&req->tl_requests == &connection->transfer_log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) req = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) connection->req_next = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static void set_if_null_req_ack_pending(struct drbd_peer_device *peer_device, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (!connection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (connection->req_ack_pending == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) connection->req_ack_pending = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static void advance_conn_req_ack_pending(struct drbd_peer_device *peer_device, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) if (!connection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (connection->req_ack_pending != req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) const unsigned s = req->rq_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) if ((s & RQ_NET_SENT) && (s & RQ_NET_PENDING))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (&req->tl_requests == &connection->transfer_log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) req = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) connection->req_ack_pending = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) static void set_if_null_req_not_net_done(struct drbd_peer_device *peer_device, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if (!connection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (connection->req_not_net_done == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) connection->req_not_net_done = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) static void advance_conn_req_not_net_done(struct drbd_peer_device *peer_device, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (!connection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (connection->req_not_net_done != req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) const unsigned s = req->rq_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if ((s & RQ_NET_SENT) && !(s & RQ_NET_DONE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) if (&req->tl_requests == &connection->transfer_log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) req = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) connection->req_not_net_done = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /* I'd like this to be the only place that manipulates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * req->completion_ref and req->kref. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) static void mod_rq_state(struct drbd_request *req, struct bio_and_error *m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) int clear, int set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) struct drbd_device *device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) struct drbd_peer_device *peer_device = first_peer_device(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) unsigned s = req->rq_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) int c_put = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (drbd_suspended(device) && !((s | clear) & RQ_COMPLETION_SUSP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) set |= RQ_COMPLETION_SUSP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) /* apply */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) req->rq_state &= ~clear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) req->rq_state |= set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) /* no change? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) if (req->rq_state == s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) /* intent: get references */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) kref_get(&req->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (!(s & RQ_LOCAL_PENDING) && (set & RQ_LOCAL_PENDING))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) atomic_inc(&req->completion_ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (!(s & RQ_NET_PENDING) && (set & RQ_NET_PENDING)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) inc_ap_pending(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) atomic_inc(&req->completion_ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) if (!(s & RQ_NET_QUEUED) && (set & RQ_NET_QUEUED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) atomic_inc(&req->completion_ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) set_if_null_req_next(peer_device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (!(s & RQ_EXP_BARR_ACK) && (set & RQ_EXP_BARR_ACK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) kref_get(&req->kref); /* wait for the DONE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (!(s & RQ_NET_SENT) && (set & RQ_NET_SENT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) /* potentially already completed in the ack_receiver thread */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (!(s & RQ_NET_DONE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) atomic_add(req->i.size >> 9, &device->ap_in_flight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) set_if_null_req_not_net_done(peer_device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) if (req->rq_state & RQ_NET_PENDING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) set_if_null_req_ack_pending(peer_device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) if (!(s & RQ_COMPLETION_SUSP) && (set & RQ_COMPLETION_SUSP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) atomic_inc(&req->completion_ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) /* progress: put references */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if ((s & RQ_COMPLETION_SUSP) && (clear & RQ_COMPLETION_SUSP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) ++c_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (!(s & RQ_LOCAL_ABORTED) && (set & RQ_LOCAL_ABORTED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) D_ASSERT(device, req->rq_state & RQ_LOCAL_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) ++c_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if ((s & RQ_LOCAL_PENDING) && (clear & RQ_LOCAL_PENDING)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) if (req->rq_state & RQ_LOCAL_ABORTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) kref_put(&req->kref, drbd_req_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) ++c_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) list_del_init(&req->req_pending_local);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) if ((s & RQ_NET_PENDING) && (clear & RQ_NET_PENDING)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) dec_ap_pending(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) ++c_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) req->acked_jif = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) advance_conn_req_ack_pending(peer_device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if ((s & RQ_NET_QUEUED) && (clear & RQ_NET_QUEUED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) ++c_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) advance_conn_req_next(peer_device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (!(s & RQ_NET_DONE) && (set & RQ_NET_DONE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (s & RQ_NET_SENT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) atomic_sub(req->i.size >> 9, &device->ap_in_flight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (s & RQ_EXP_BARR_ACK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) kref_put(&req->kref, drbd_req_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) req->net_done_jif = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) /* in ahead/behind mode, or just in case,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * before we finally destroy this request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * the caching pointers must not reference it anymore */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) advance_conn_req_next(peer_device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) advance_conn_req_ack_pending(peer_device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) advance_conn_req_not_net_done(peer_device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) /* potentially complete and destroy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) /* If we made progress, retry conflicting peer requests, if any. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (req->i.waiting)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) wake_up(&device->misc_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) drbd_req_put_completion_ref(req, m, c_put);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) kref_put(&req->kref, drbd_req_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) static void drbd_report_io_error(struct drbd_device *device, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (!__ratelimit(&drbd_ratelimit_state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) drbd_warn(device, "local %s IO error sector %llu+%u on %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) (req->rq_state & RQ_WRITE) ? "WRITE" : "READ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) (unsigned long long)req->i.sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) req->i.size >> 9,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) bdevname(device->ldev->backing_bdev, b));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) /* Helper for HANDED_OVER_TO_NETWORK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) * Is this a protocol A write (neither WRITE_ACK nor RECEIVE_ACK expected)?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * Is it also still "PENDING"?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * --> If so, clear PENDING and set NET_OK below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * If it is a protocol A write, but not RQ_PENDING anymore, neg-ack was faster
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) * (and we must not set RQ_NET_OK) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) static inline bool is_pending_write_protocol_A(struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return (req->rq_state &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) (RQ_WRITE|RQ_NET_PENDING|RQ_EXP_WRITE_ACK|RQ_EXP_RECEIVE_ACK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) == (RQ_WRITE|RQ_NET_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) /* obviously this could be coded as many single functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * instead of one huge switch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * or by putting the code directly in the respective locations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * (as it has been before).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) * but having it this way
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * enforces that it is all in this one place, where it is easier to audit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * it makes it obvious that whatever "event" "happens" to a request should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) * happen "atomically" within the req_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * and it enforces that we have to think in a very structured manner
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * about the "events" that may happen to a request during its life time ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) int __req_mod(struct drbd_request *req, enum drbd_req_event what,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) struct bio_and_error *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) struct drbd_device *const device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) struct drbd_peer_device *const peer_device = first_peer_device(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) struct net_conf *nc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) int p, rv = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) if (m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) m->bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) switch (what) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) drbd_err(device, "LOGIC BUG in %s:%u\n", __FILE__ , __LINE__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) /* does not happen...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * initialization done in drbd_req_new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) case CREATED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) case TO_BE_SENT: /* via network */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /* reached via __drbd_make_request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * and from w_read_retry_remote */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) D_ASSERT(device, !(req->rq_state & RQ_NET_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) nc = rcu_dereference(connection->net_conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) p = nc->wire_protocol;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) req->rq_state |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) p == DRBD_PROT_C ? RQ_EXP_WRITE_ACK :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) p == DRBD_PROT_B ? RQ_EXP_RECEIVE_ACK : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) mod_rq_state(req, m, 0, RQ_NET_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) case TO_BE_SUBMITTED: /* locally */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) /* reached via __drbd_make_request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) D_ASSERT(device, !(req->rq_state & RQ_LOCAL_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) mod_rq_state(req, m, 0, RQ_LOCAL_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) case COMPLETED_OK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (req->rq_state & RQ_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) device->writ_cnt += req->i.size >> 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) device->read_cnt += req->i.size >> 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) mod_rq_state(req, m, RQ_LOCAL_PENDING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) RQ_LOCAL_COMPLETED|RQ_LOCAL_OK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) case ABORT_DISK_IO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) mod_rq_state(req, m, 0, RQ_LOCAL_ABORTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) case WRITE_COMPLETED_WITH_ERROR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) drbd_report_io_error(device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) __drbd_chk_io_error(device, DRBD_WRITE_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) case READ_COMPLETED_WITH_ERROR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) drbd_set_out_of_sync(device, req->i.sector, req->i.size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) drbd_report_io_error(device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) __drbd_chk_io_error(device, DRBD_READ_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) case READ_AHEAD_COMPLETED_WITH_ERROR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) /* it is legal to fail read-ahead, no __drbd_chk_io_error in that case. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) case DISCARD_COMPLETED_NOTSUPP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) case DISCARD_COMPLETED_WITH_ERROR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) /* I'd rather not detach from local disk just because it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * failed a REQ_OP_DISCARD. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) case QUEUE_FOR_NET_READ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) /* READ, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * no local disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * or target area marked as invalid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * or just got an io-error. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) /* from __drbd_make_request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * or from bio_endio during read io-error recovery */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) /* So we can verify the handle in the answer packet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * Corresponding drbd_remove_request_interval is in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * drbd_req_complete() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) D_ASSERT(device, drbd_interval_empty(&req->i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) drbd_insert_interval(&device->read_requests, &req->i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) set_bit(UNPLUG_REMOTE, &device->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) D_ASSERT(device, (req->rq_state & RQ_LOCAL_MASK) == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) mod_rq_state(req, m, 0, RQ_NET_QUEUED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) req->w.cb = w_send_read_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) drbd_queue_work(&connection->sender_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) &req->w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) case QUEUE_FOR_NET_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) /* assert something? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) /* from __drbd_make_request only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) /* Corresponding drbd_remove_request_interval is in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * drbd_req_complete() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) D_ASSERT(device, drbd_interval_empty(&req->i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) drbd_insert_interval(&device->write_requests, &req->i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) /* NOTE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * In case the req ended up on the transfer log before being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * queued on the worker, it could lead to this request being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * missed during cleanup after connection loss.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * So we have to do both operations here,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) * within the same lock that protects the transfer log.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * _req_add_to_epoch(req); this has to be after the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * _maybe_start_new_epoch(req); which happened in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * __drbd_make_request, because we now may set the bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * again ourselves to close the current epoch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * Add req to the (now) current epoch (barrier). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) /* otherwise we may lose an unplug, which may cause some remote
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * io-scheduler timeout to expire, increasing maximum latency,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * hurting performance. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) set_bit(UNPLUG_REMOTE, &device->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) /* queue work item to send data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) mod_rq_state(req, m, 0, RQ_NET_QUEUED|RQ_EXP_BARR_ACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) req->w.cb = w_send_dblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) drbd_queue_work(&connection->sender_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) &req->w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) /* close the epoch, in case it outgrew the limit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) nc = rcu_dereference(connection->net_conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) p = nc->max_epoch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) if (connection->current_tle_writes >= p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) start_new_tl_epoch(connection);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) case QUEUE_FOR_SEND_OOS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) mod_rq_state(req, m, 0, RQ_NET_QUEUED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) req->w.cb = w_send_out_of_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) drbd_queue_work(&connection->sender_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) &req->w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) case READ_RETRY_REMOTE_CANCELED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) case SEND_CANCELED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) case SEND_FAILED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) /* real cleanup will be done from tl_clear. just update flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * so it is no longer marked as on the worker queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) mod_rq_state(req, m, RQ_NET_QUEUED, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) case HANDED_OVER_TO_NETWORK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) /* assert something? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) if (is_pending_write_protocol_A(req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) /* this is what is dangerous about protocol A:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) * pretend it was successfully written on the peer. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) mod_rq_state(req, m, RQ_NET_QUEUED|RQ_NET_PENDING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) RQ_NET_SENT|RQ_NET_OK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) mod_rq_state(req, m, RQ_NET_QUEUED, RQ_NET_SENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) /* It is still not yet RQ_NET_DONE until the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * corresponding epoch barrier got acked as well,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) * so we know what to dirty on connection loss. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) case OOS_HANDED_TO_NETWORK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) /* Was not set PENDING, no longer QUEUED, so is now DONE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) * as far as this connection is concerned. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) mod_rq_state(req, m, RQ_NET_QUEUED, RQ_NET_DONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) case CONNECTION_LOST_WHILE_PENDING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) /* transfer log cleanup after connection loss */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) mod_rq_state(req, m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) RQ_NET_OK|RQ_NET_PENDING|RQ_COMPLETION_SUSP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) RQ_NET_DONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) case CONFLICT_RESOLVED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) /* for superseded conflicting writes of multiple primaries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) * there is no need to keep anything in the tl, potential
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * node crashes are covered by the activity log.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * If this request had been marked as RQ_POSTPONED before,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * it will actually not be completed, but "restarted",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * resubmitted from the retry worker context. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) D_ASSERT(device, req->rq_state & RQ_EXP_WRITE_ACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) mod_rq_state(req, m, RQ_NET_PENDING, RQ_NET_DONE|RQ_NET_OK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) case WRITE_ACKED_BY_PEER_AND_SIS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) req->rq_state |= RQ_NET_SIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) case WRITE_ACKED_BY_PEER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) /* Normal operation protocol C: successfully written on peer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * During resync, even in protocol != C,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * we requested an explicit write ack anyways.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * Which means we cannot even assert anything here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * Nothing more to do here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * We want to keep the tl in place for all protocols, to cater
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * for volatile write-back caches on lower level devices. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) goto ack_common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) case RECV_ACKED_BY_PEER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) D_ASSERT(device, req->rq_state & RQ_EXP_RECEIVE_ACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) /* protocol B; pretends to be successfully written on peer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) * see also notes above in HANDED_OVER_TO_NETWORK about
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * protocol != C */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) ack_common:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) mod_rq_state(req, m, RQ_NET_PENDING, RQ_NET_OK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) case POSTPONE_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) D_ASSERT(device, req->rq_state & RQ_EXP_WRITE_ACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) /* If this node has already detected the write conflict, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * worker will be waiting on misc_wait. Wake it up once this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * request has completed locally.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) req->rq_state |= RQ_POSTPONED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) if (req->i.waiting)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) wake_up(&device->misc_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) /* Do not clear RQ_NET_PENDING. This request will make further
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * progress via restart_conflicting_writes() or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * fail_postponed_requests(). Hopefully. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) case NEG_ACKED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) mod_rq_state(req, m, RQ_NET_OK|RQ_NET_PENDING, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) case FAIL_FROZEN_DISK_IO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) if (!(req->rq_state & RQ_LOCAL_COMPLETED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) mod_rq_state(req, m, RQ_COMPLETION_SUSP, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) case RESTART_FROZEN_DISK_IO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) if (!(req->rq_state & RQ_LOCAL_COMPLETED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) mod_rq_state(req, m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) RQ_COMPLETION_SUSP|RQ_LOCAL_COMPLETED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) RQ_LOCAL_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) rv = MR_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) if (bio_data_dir(req->master_bio) == WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) rv = MR_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) get_ldev(device); /* always succeeds in this call path */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) req->w.cb = w_restart_disk_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) drbd_queue_work(&connection->sender_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) &req->w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) case RESEND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) /* Simply complete (local only) READs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) if (!(req->rq_state & RQ_WRITE) && !req->w.cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) mod_rq_state(req, m, RQ_COMPLETION_SUSP, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) /* If RQ_NET_OK is already set, we got a P_WRITE_ACK or P_RECV_ACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) before the connection loss (B&C only); only P_BARRIER_ACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) (or the local completion?) was missing when we suspended.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) Throwing them out of the TL here by pretending we got a BARRIER_ACK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) During connection handshake, we ensure that the peer was not rebooted. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if (!(req->rq_state & RQ_NET_OK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) /* FIXME could this possibly be a req->dw.cb == w_send_out_of_sync?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) * in that case we must not set RQ_NET_PENDING. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) mod_rq_state(req, m, RQ_COMPLETION_SUSP, RQ_NET_QUEUED|RQ_NET_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) if (req->w.cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) /* w.cb expected to be w_send_dblock, or w_send_read_req */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) drbd_queue_work(&connection->sender_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) &req->w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) rv = req->rq_state & RQ_WRITE ? MR_WRITE : MR_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) } /* else: FIXME can this happen? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) fallthrough; /* to BARRIER_ACKED */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) case BARRIER_ACKED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) /* barrier ack for READ requests does not make sense */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) if (!(req->rq_state & RQ_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (req->rq_state & RQ_NET_PENDING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) /* barrier came in before all requests were acked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) * this is bad, because if the connection is lost now,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) * we won't be able to clean them up... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) drbd_err(device, "FIXME (BARRIER_ACKED but pending)\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) /* Allowed to complete requests, even while suspended.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) * As this is called for all requests within a matching epoch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) * we need to filter, and only set RQ_NET_DONE for those that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) * have actually been on the wire. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) mod_rq_state(req, m, RQ_COMPLETION_SUSP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) (req->rq_state & RQ_NET_MASK) ? RQ_NET_DONE : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) case DATA_RECEIVED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) mod_rq_state(req, m, RQ_NET_PENDING, RQ_NET_OK|RQ_NET_DONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) case QUEUE_AS_DRBD_BARRIER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) start_new_tl_epoch(connection);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) mod_rq_state(req, m, 0, RQ_NET_OK|RQ_NET_DONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) return rv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) /* we may do a local read if:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * - we are consistent (of course),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * - or we are generally inconsistent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * BUT we are still/already IN SYNC for this area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * since size may be bigger than BM_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * we may need to check several bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) static bool drbd_may_do_local_read(struct drbd_device *device, sector_t sector, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) unsigned long sbnr, ebnr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) sector_t esector, nr_sectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) if (device->state.disk == D_UP_TO_DATE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (device->state.disk != D_INCONSISTENT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) esector = sector + (size >> 9) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) nr_sectors = get_capacity(device->vdisk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) D_ASSERT(device, sector < nr_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) D_ASSERT(device, esector < nr_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) sbnr = BM_SECT_TO_BIT(sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) ebnr = BM_SECT_TO_BIT(esector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) return drbd_bm_count_bits(device, sbnr, ebnr) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) static bool remote_due_to_read_balancing(struct drbd_device *device, sector_t sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) enum drbd_read_balancing rbm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) struct backing_dev_info *bdi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) int stripe_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) switch (rbm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) case RB_CONGESTED_REMOTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) bdi = device->ldev->backing_bdev->bd_disk->queue->backing_dev_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) return bdi_read_congested(bdi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) case RB_LEAST_PENDING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) return atomic_read(&device->local_cnt) >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) atomic_read(&device->ap_pending_cnt) + atomic_read(&device->rs_pending_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) case RB_32K_STRIPING: /* stripe_shift = 15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) case RB_64K_STRIPING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) case RB_128K_STRIPING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) case RB_256K_STRIPING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) case RB_512K_STRIPING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) case RB_1M_STRIPING: /* stripe_shift = 20 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) stripe_shift = (rbm - RB_32K_STRIPING + 15);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) return (sector >> (stripe_shift - 9)) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) case RB_ROUND_ROBIN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) return test_and_change_bit(READ_BALANCE_RR, &device->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) case RB_PREFER_REMOTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) case RB_PREFER_LOCAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) * complete_conflicting_writes - wait for any conflicting write requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) * The write_requests tree contains all active write requests which we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) * currently know about. Wait for any requests to complete which conflict with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) * the new one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * Only way out: remove the conflicting intervals from the tree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) static void complete_conflicting_writes(struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) DEFINE_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) struct drbd_device *device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) struct drbd_interval *i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) sector_t sector = req->i.sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) int size = req->i.size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) drbd_for_each_overlap(i, &device->write_requests, sector, size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) /* Ignore, if already completed to upper layers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) if (i->completed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) /* Handle the first found overlap. After the schedule
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) * we have to restart the tree walk. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) if (!i) /* if any */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) /* Indicate to wake up device->misc_wait on progress. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) prepare_to_wait(&device->misc_wait, &wait, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) i->waiting = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) spin_unlock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) spin_lock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) finish_wait(&device->misc_wait, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) /* called within req_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) static void maybe_pull_ahead(struct drbd_device *device)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) struct drbd_connection *connection = first_peer_device(device)->connection;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) struct net_conf *nc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) bool congested = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) enum drbd_on_congestion on_congestion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) nc = rcu_dereference(connection->net_conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) on_congestion = nc ? nc->on_congestion : OC_BLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) if (on_congestion == OC_BLOCK ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) connection->agreed_pro_version < 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) if (on_congestion == OC_PULL_AHEAD && device->state.conn == C_AHEAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) return; /* nothing to do ... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /* If I don't even have good local storage, we can not reasonably try
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) * to pull ahead of the peer. We also need the local reference to make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) * sure device->act_log is there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) if (!get_ldev_if_state(device, D_UP_TO_DATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) if (nc->cong_fill &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) atomic_read(&device->ap_in_flight) >= nc->cong_fill) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) drbd_info(device, "Congestion-fill threshold reached\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) congested = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) if (device->act_log->used >= nc->cong_extents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) drbd_info(device, "Congestion-extents threshold reached\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) congested = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) if (congested) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) /* start a new epoch for non-mirrored writes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) start_new_tl_epoch(first_peer_device(device)->connection);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) if (on_congestion == OC_PULL_AHEAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) _drbd_set_state(_NS(device, conn, C_AHEAD), 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) else /*nc->on_congestion == OC_DISCONNECT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) _drbd_set_state(_NS(device, conn, C_DISCONNECTING), 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) put_ldev(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) /* If this returns false, and req->private_bio is still set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) * this should be submitted locally.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) * If it returns false, but req->private_bio is not set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) * we do not have access to good data :(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) * Otherwise, this destroys req->private_bio, if any,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) * and returns true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) static bool do_remote_read(struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) struct drbd_device *device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) enum drbd_read_balancing rbm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) if (req->private_bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) if (!drbd_may_do_local_read(device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) req->i.sector, req->i.size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) bio_put(req->private_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) req->private_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) put_ldev(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) if (device->state.pdsk != D_UP_TO_DATE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) if (req->private_bio == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) /* TODO: improve read balancing decisions, take into account drbd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) * protocol, pending requests etc. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) rbm = rcu_dereference(device->ldev->disk_conf)->read_balancing;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (rbm == RB_PREFER_LOCAL && req->private_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) return false; /* submit locally */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) if (remote_due_to_read_balancing(device, req->i.sector, rbm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) if (req->private_bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) bio_put(req->private_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) req->private_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) put_ldev(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) bool drbd_should_do_remote(union drbd_dev_state s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) return s.pdsk == D_UP_TO_DATE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) (s.pdsk >= D_INCONSISTENT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) s.conn >= C_WF_BITMAP_T &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) s.conn < C_AHEAD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) /* Before proto 96 that was >= CONNECTED instead of >= C_WF_BITMAP_T.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) That is equivalent since before 96 IO was frozen in the C_WF_BITMAP*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) states. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) static bool drbd_should_send_out_of_sync(union drbd_dev_state s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) return s.conn == C_AHEAD || s.conn == C_WF_BITMAP_S;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) /* pdsk = D_INCONSISTENT as a consequence. Protocol 96 check not necessary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) since we enter state C_AHEAD only if proto >= 96 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) /* returns number of connections (== 1, for drbd 8.4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * expected to actually write this data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * which does NOT include those that we are L_AHEAD for. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) static int drbd_process_write_request(struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) struct drbd_device *device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) int remote, send_oos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) remote = drbd_should_do_remote(device->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) send_oos = drbd_should_send_out_of_sync(device->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) /* Need to replicate writes. Unless it is an empty flush,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) * which is better mapped to a DRBD P_BARRIER packet,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) * also for drbd wire protocol compatibility reasons.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) * If this was a flush, just start a new epoch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) * Unless the current epoch was empty anyways, or we are not currently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) * replicating, in which case there is no point. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) if (unlikely(req->i.size == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) /* The only size==0 bios we expect are empty flushes. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) D_ASSERT(device, req->master_bio->bi_opf & REQ_PREFLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) if (remote)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) _req_mod(req, QUEUE_AS_DRBD_BARRIER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) return remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) if (!remote && !send_oos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) D_ASSERT(device, !(remote && send_oos));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) if (remote) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) _req_mod(req, TO_BE_SENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) _req_mod(req, QUEUE_FOR_NET_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) } else if (drbd_set_out_of_sync(device, req->i.sector, req->i.size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) _req_mod(req, QUEUE_FOR_SEND_OOS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) return remote;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) static void drbd_process_discard_or_zeroes_req(struct drbd_request *req, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) int err = drbd_issue_discard_or_zero_out(req->device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) req->i.sector, req->i.size >> 9, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) req->private_bio->bi_status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) bio_endio(req->private_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) drbd_submit_req_private_bio(struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) struct drbd_device *device = req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) struct bio *bio = req->private_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) unsigned int type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) if (bio_op(bio) != REQ_OP_READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) type = DRBD_FAULT_DT_WR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) else if (bio->bi_opf & REQ_RAHEAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) type = DRBD_FAULT_DT_RA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) type = DRBD_FAULT_DT_RD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) bio_set_dev(bio, device->ldev->backing_bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) /* State may have changed since we grabbed our reference on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) * ->ldev member. Double check, and short-circuit to endio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) * In case the last activity log transaction failed to get on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) * stable storage, and this is a WRITE, we may not even submit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) * this bio. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) if (get_ldev(device)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) if (drbd_insert_fault(device, type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) else if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) drbd_process_discard_or_zeroes_req(req, EE_ZEROOUT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) ((bio->bi_opf & REQ_NOUNMAP) ? 0 : EE_TRIM));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) else if (bio_op(bio) == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) drbd_process_discard_or_zeroes_req(req, EE_TRIM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) put_ldev(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) static void drbd_queue_write(struct drbd_device *device, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) spin_lock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) list_add_tail(&req->tl_requests, &device->submit.writes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) list_add_tail(&req->req_pending_master_completion,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) &device->pending_master_completion[1 /* WRITE */]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) spin_unlock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) queue_work(device->submit.wq, &device->submit.worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) /* do_submit() may sleep internally on al_wait, too */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) wake_up(&device->al_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) /* returns the new drbd_request pointer, if the caller is expected to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) * drbd_send_and_submit() it (to save latency), or NULL if we queued the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) * request on the submitter thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) * Returns ERR_PTR(-ENOMEM) if we cannot allocate a drbd_request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) static struct drbd_request *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) drbd_request_prepare(struct drbd_device *device, struct bio *bio, unsigned long start_jif)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) const int rw = bio_data_dir(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) struct drbd_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) /* allocate outside of all locks; */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) req = drbd_req_new(device, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) if (!req) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) dec_ap_bio(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) /* only pass the error to the upper layers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) * if user cannot handle io errors, that's not our business. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) drbd_err(device, "could not kmalloc() req\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) bio->bi_status = BLK_STS_RESOURCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) /* Update disk stats */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) req->start_jif = bio_start_io_acct(req->master_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) if (!get_ldev(device)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) bio_put(req->private_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) req->private_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) /* process discards always from our submitter thread */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) if (bio_op(bio) == REQ_OP_WRITE_ZEROES ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) bio_op(bio) == REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) goto queue_for_submitter_thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) if (rw == WRITE && req->private_bio && req->i.size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) && !test_bit(AL_SUSPENDED, &device->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) if (!drbd_al_begin_io_fastpath(device, &req->i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) goto queue_for_submitter_thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) req->rq_state |= RQ_IN_ACT_LOG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) req->in_actlog_jif = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) return req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) queue_for_submitter_thread:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) atomic_inc(&device->ap_actlog_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) drbd_queue_write(device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) /* Require at least one path to current data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * We don't want to allow writes on C_STANDALONE D_INCONSISTENT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) * We would not allow to read what was written,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) * we would not have bumped the data generation uuids,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) * we would cause data divergence for all the wrong reasons.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) * If we don't see at least one D_UP_TO_DATE, we will fail this request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) * which either returns EIO, or, if OND_SUSPEND_IO is set, suspends IO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) * and queues for retry later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) static bool may_do_writes(struct drbd_device *device)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) const union drbd_dev_state s = device->state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) return s.disk == D_UP_TO_DATE || s.pdsk == D_UP_TO_DATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) struct drbd_plug_cb {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) struct blk_plug_cb cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) struct drbd_request *most_recent_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) /* do we need more? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) static void drbd_unplug(struct blk_plug_cb *cb, bool from_schedule)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) struct drbd_plug_cb *plug = container_of(cb, struct drbd_plug_cb, cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) struct drbd_resource *resource = plug->cb.data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) struct drbd_request *req = plug->most_recent_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) kfree(cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) if (!req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) spin_lock_irq(&resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) /* In case the sender did not process it yet, raise the flag to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) * have it followed with P_UNPLUG_REMOTE just after. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) req->rq_state |= RQ_UNPLUG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) /* but also queue a generic unplug */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) drbd_queue_unplug(req->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) kref_put(&req->kref, drbd_req_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) spin_unlock_irq(&resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) static struct drbd_plug_cb* drbd_check_plugged(struct drbd_resource *resource)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) /* A lot of text to say
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) * return (struct drbd_plug_cb*)blk_check_plugged(); */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) struct drbd_plug_cb *plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) struct blk_plug_cb *cb = blk_check_plugged(drbd_unplug, resource, sizeof(*plug));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) if (cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) plug = container_of(cb, struct drbd_plug_cb, cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) plug = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) return plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) static void drbd_update_plug(struct drbd_plug_cb *plug, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) struct drbd_request *tmp = plug->most_recent_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) /* Will be sent to some peer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) * Remember to tag it with UNPLUG_REMOTE on unplug */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) kref_get(&req->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) plug->most_recent_req = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) if (tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) kref_put(&tmp->kref, drbd_req_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) struct drbd_resource *resource = device->resource;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) const int rw = bio_data_dir(req->master_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) struct bio_and_error m = { NULL, };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) bool no_remote = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) bool submit_private_bio = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) spin_lock_irq(&resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) if (rw == WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) /* This may temporarily give up the req_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) * but will re-aquire it before it returns here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) * Needs to be before the check on drbd_suspended() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) complete_conflicting_writes(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) /* no more giving up req_lock from now on! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) /* check for congestion, and potentially stop sending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) * full data updates, but start sending "dirty bits" only. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) maybe_pull_ahead(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) if (drbd_suspended(device)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) /* push back and retry: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) req->rq_state |= RQ_POSTPONED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) if (req->private_bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) bio_put(req->private_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) req->private_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) put_ldev(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) /* We fail READ early, if we can not serve it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) * We must do this before req is registered on any lists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) * Otherwise, drbd_req_complete() will queue failed READ for retry. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) if (rw != WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) if (!do_remote_read(req) && !req->private_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) goto nodata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) /* which transfer log epoch does this belong to? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) req->epoch = atomic_read(&first_peer_device(device)->connection->current_tle_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) /* no point in adding empty flushes to the transfer log,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) * they are mapped to drbd barriers already. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) if (likely(req->i.size!=0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) if (rw == WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) first_peer_device(device)->connection->current_tle_writes++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) list_add_tail(&req->tl_requests, &first_peer_device(device)->connection->transfer_log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) if (rw == WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) if (req->private_bio && !may_do_writes(device)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) bio_put(req->private_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) req->private_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) put_ldev(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) goto nodata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) if (!drbd_process_write_request(req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) no_remote = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) /* We either have a private_bio, or we can read from remote.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) * Otherwise we had done the goto nodata above. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) if (req->private_bio == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) _req_mod(req, TO_BE_SENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) _req_mod(req, QUEUE_FOR_NET_READ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) no_remote = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) if (no_remote == false) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) struct drbd_plug_cb *plug = drbd_check_plugged(resource);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) if (plug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) drbd_update_plug(plug, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) /* If it took the fast path in drbd_request_prepare, add it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) * The slow path has added it already. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) if (list_empty(&req->req_pending_master_completion))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) list_add_tail(&req->req_pending_master_completion,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) &device->pending_master_completion[rw == WRITE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) if (req->private_bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) /* needs to be marked within the same spinlock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) req->pre_submit_jif = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) list_add_tail(&req->req_pending_local,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) &device->pending_completion[rw == WRITE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) _req_mod(req, TO_BE_SUBMITTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) /* but we need to give up the spinlock to submit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) submit_private_bio = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) } else if (no_remote) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) nodata:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) if (__ratelimit(&drbd_ratelimit_state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) drbd_err(device, "IO ERROR: neither local nor remote data, sector %llu+%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) (unsigned long long)req->i.sector, req->i.size >> 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) /* A write may have been queued for send_oos, however.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) * So we can not simply free it, we must go through drbd_req_put_completion_ref() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) drbd_req_put_completion_ref(req, &m, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) spin_unlock_irq(&resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) /* Even though above is a kref_put(), this is safe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) * As long as we still need to submit our private bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) * we hold a completion ref, and the request cannot disappear.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) * If however this request did not even have a private bio to submit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) * (e.g. remote read), req may already be invalid now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) * That's why we cannot check on req->private_bio. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) if (submit_private_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) drbd_submit_req_private_bio(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) if (m.bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) complete_master_bio(device, &m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) void __drbd_make_request(struct drbd_device *device, struct bio *bio, unsigned long start_jif)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) struct drbd_request *req = drbd_request_prepare(device, bio, start_jif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) if (IS_ERR_OR_NULL(req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) drbd_send_and_submit(device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) static void submit_fast_path(struct drbd_device *device, struct list_head *incoming)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) struct drbd_request *req, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) list_for_each_entry_safe(req, tmp, incoming, tl_requests) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) const int rw = bio_data_dir(req->master_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) if (rw == WRITE /* rw != WRITE should not even end up here! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) && req->private_bio && req->i.size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) && !test_bit(AL_SUSPENDED, &device->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) if (!drbd_al_begin_io_fastpath(device, &req->i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) req->rq_state |= RQ_IN_ACT_LOG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) req->in_actlog_jif = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) atomic_dec(&device->ap_actlog_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) list_del_init(&req->tl_requests);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) drbd_send_and_submit(device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) static bool prepare_al_transaction_nonblock(struct drbd_device *device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) struct list_head *incoming,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) struct list_head *pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) struct list_head *later)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) struct drbd_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) int wake = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) spin_lock_irq(&device->al_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) while ((req = list_first_entry_or_null(incoming, struct drbd_request, tl_requests))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) err = drbd_al_begin_io_nonblock(device, &req->i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) if (err == -ENOBUFS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) if (err == -EBUSY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) wake = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) list_move_tail(&req->tl_requests, later);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) list_move_tail(&req->tl_requests, pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) spin_unlock_irq(&device->al_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (wake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) wake_up(&device->al_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) return !list_empty(pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) static void send_and_submit_pending(struct drbd_device *device, struct list_head *pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) struct drbd_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) while ((req = list_first_entry_or_null(pending, struct drbd_request, tl_requests))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) req->rq_state |= RQ_IN_ACT_LOG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) req->in_actlog_jif = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) atomic_dec(&device->ap_actlog_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) list_del_init(&req->tl_requests);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) drbd_send_and_submit(device, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) void do_submit(struct work_struct *ws)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) struct drbd_device *device = container_of(ws, struct drbd_device, submit.worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) LIST_HEAD(incoming); /* from drbd_make_request() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) LIST_HEAD(pending); /* to be submitted after next AL-transaction commit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) LIST_HEAD(busy); /* blocked by resync requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) /* grab new incoming requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) spin_lock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) list_splice_tail_init(&device->submit.writes, &incoming);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) spin_unlock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) DEFINE_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) /* move used-to-be-busy back to front of incoming */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) list_splice_init(&busy, &incoming);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) submit_fast_path(device, &incoming);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) if (list_empty(&incoming))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) prepare_to_wait(&device->al_wait, &wait, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) list_splice_init(&busy, &incoming);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) prepare_al_transaction_nonblock(device, &incoming, &pending, &busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) if (!list_empty(&pending))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) /* If all currently "hot" activity log extents are kept busy by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) * incoming requests, we still must not totally starve new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) * requests to "cold" extents.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) * Something left on &incoming means there had not been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) * enough update slots available, and the activity log
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) * has been marked as "starving".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) * Try again now, without looking for new requests,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) * effectively blocking all new requests until we made
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) * at least _some_ progress with what we currently have.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) if (!list_empty(&incoming))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) /* Nothing moved to pending, but nothing left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) * on incoming: all moved to busy!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) * Grab new and iterate. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) spin_lock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) list_splice_tail_init(&device->submit.writes, &incoming);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) spin_unlock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) finish_wait(&device->al_wait, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) /* If the transaction was full, before all incoming requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) * had been processed, skip ahead to commit, and iterate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) * without splicing in more incoming requests from upper layers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) * Else, if all incoming have been processed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) * they have become either "pending" (to be submitted after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) * next transaction commit) or "busy" (blocked by resync).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) * Maybe more was queued, while we prepared the transaction?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) * Try to stuff those into this transaction as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) * Be strictly non-blocking here,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) * we already have something to commit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) * Commit if we don't make any more progres.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) while (list_empty(&incoming)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) LIST_HEAD(more_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) LIST_HEAD(more_incoming);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) bool made_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) /* It is ok to look outside the lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) * it's only an optimization anyways */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) if (list_empty(&device->submit.writes))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) spin_lock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) list_splice_tail_init(&device->submit.writes, &more_incoming);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) spin_unlock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) if (list_empty(&more_incoming))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) made_progress = prepare_al_transaction_nonblock(device, &more_incoming, &more_pending, &busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) list_splice_tail_init(&more_pending, &pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) list_splice_tail_init(&more_incoming, &incoming);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) if (!made_progress)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) drbd_al_begin_io_commit(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) send_and_submit_pending(device, &pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) blk_qc_t drbd_submit_bio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) struct drbd_device *device = bio->bi_disk->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) unsigned long start_jif;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) blk_queue_split(&bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) start_jif = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) * what we "blindly" assume:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) D_ASSERT(device, IS_ALIGNED(bio->bi_iter.bi_size, 512));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) inc_ap_bio(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) __drbd_make_request(device, bio, start_jif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) return BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) static bool net_timeout_reached(struct drbd_request *net_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) struct drbd_connection *connection,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) unsigned long now, unsigned long ent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) unsigned int ko_count, unsigned int timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) struct drbd_device *device = net_req->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) if (!time_after(now, net_req->pre_send_jif + ent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) if (time_in_range(now, connection->last_reconnect_jif, connection->last_reconnect_jif + ent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) if (net_req->rq_state & RQ_NET_PENDING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) drbd_warn(device, "Remote failed to finish a request within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) /* We received an ACK already (or are using protocol A),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) * but are waiting for the epoch closing barrier ack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) * Check if we sent the barrier already. We should not blame the peer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) * for being unresponsive, if we did not even ask it yet. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) if (net_req->epoch == connection->send.current_epoch_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) drbd_warn(device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) "We did not send a P_BARRIER for %ums > ko-count (%u) * timeout (%u * 0.1s); drbd kernel thread blocked?\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) /* Worst case: we may have been blocked for whatever reason, then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) * suddenly are able to send a lot of requests (and epoch separating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) * barriers) in quick succession.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) * The timestamp of the net_req may be much too old and not correspond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) * to the sending time of the relevant unack'ed barrier packet, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) * would trigger a spurious timeout. The latest barrier packet may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) * have a too recent timestamp to trigger the timeout, potentially miss
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) * a timeout. Right now we don't have a place to conveniently store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) * these timestamps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) * But in this particular situation, the application requests are still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) * completed to upper layers, DRBD should still "feel" responsive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) * No need yet to kill this connection, it may still recover.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) * If not, eventually we will have queued enough into the network for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) * us to block. From that point of view, the timestamp of the last sent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) * barrier packet is relevant enough.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) if (time_after(now, connection->send.last_sent_barrier_jif + ent)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) drbd_warn(device, "Remote failed to answer a P_BARRIER (sent at %lu jif; now=%lu jif) within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) connection->send.last_sent_barrier_jif, now,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) jiffies_to_msecs(now - connection->send.last_sent_barrier_jif), ko_count, timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) /* A request is considered timed out, if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) * - we have some effective timeout from the configuration,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) * with some state restrictions applied,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) * - the oldest request is waiting for a response from the network
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) * resp. the local disk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) * - the oldest request is in fact older than the effective timeout,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) * - the connection was established (resp. disk was attached)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) * for longer than the timeout already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) * Note that for 32bit jiffies and very stable connections/disks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) * we may have a wrap around, which is catched by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) * !time_in_range(now, last_..._jif, last_..._jif + timeout).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) * Side effect: once per 32bit wrap-around interval, which means every
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) * ~198 days with 250 HZ, we have a window where the timeout would need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) * to expire twice (worst case) to become effective. Good enough.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) void request_timer_fn(struct timer_list *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) struct drbd_device *device = from_timer(device, t, request_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) struct drbd_connection *connection = first_peer_device(device)->connection;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) struct drbd_request *req_read, *req_write, *req_peer; /* oldest request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) struct net_conf *nc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) unsigned long oldest_submit_jif;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) unsigned long ent = 0, dt = 0, et, nt; /* effective timeout = ko_count * timeout */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) unsigned long now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) unsigned int ko_count = 0, timeout = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) nc = rcu_dereference(connection->net_conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) if (nc && device->state.conn >= C_WF_REPORT_PARAMS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) ko_count = nc->ko_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) timeout = nc->timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) if (get_ldev(device)) { /* implicit state.disk >= D_INCONSISTENT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) dt = rcu_dereference(device->ldev->disk_conf)->disk_timeout * HZ / 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) put_ldev(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) ent = timeout * HZ/10 * ko_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) et = min_not_zero(dt, ent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) if (!et)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) return; /* Recurring timer stopped */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) now = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) nt = now + et;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) spin_lock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) req_read = list_first_entry_or_null(&device->pending_completion[0], struct drbd_request, req_pending_local);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) req_write = list_first_entry_or_null(&device->pending_completion[1], struct drbd_request, req_pending_local);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) /* maybe the oldest request waiting for the peer is in fact still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) * blocking in tcp sendmsg. That's ok, though, that's handled via the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) * socket send timeout, requesting a ping, and bumping ko-count in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) * we_should_drop_the_connection().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) /* check the oldest request we did successfully sent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) * but which is still waiting for an ACK. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) req_peer = connection->req_ack_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) /* if we don't have such request (e.g. protocoll A)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) * check the oldest requests which is still waiting on its epoch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) * closing barrier ack. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) if (!req_peer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) req_peer = connection->req_not_net_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) /* evaluate the oldest peer request only in one timer! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) if (req_peer && req_peer->device != device)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) req_peer = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) /* do we have something to evaluate? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) if (req_peer == NULL && req_write == NULL && req_read == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) oldest_submit_jif =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) (req_write && req_read)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) ? ( time_before(req_write->pre_submit_jif, req_read->pre_submit_jif)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) ? req_write->pre_submit_jif : req_read->pre_submit_jif )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) : req_write ? req_write->pre_submit_jif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) : req_read ? req_read->pre_submit_jif : now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) if (ent && req_peer && net_timeout_reached(req_peer, connection, now, ent, ko_count, timeout))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) _conn_request_state(connection, NS(conn, C_TIMEOUT), CS_VERBOSE | CS_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) if (dt && oldest_submit_jif != now &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) time_after(now, oldest_submit_jif + dt) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) !time_in_range(now, device->last_reattach_jif, device->last_reattach_jif + dt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) drbd_warn(device, "Local backing device failed to meet the disk-timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) __drbd_chk_io_error(device, DRBD_FORCE_DETACH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) /* Reschedule timer for the nearest not already expired timeout.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) * Fallback to now + min(effective network timeout, disk timeout). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) ent = (ent && req_peer && time_before(now, req_peer->pre_send_jif + ent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) ? req_peer->pre_send_jif + ent : now + et;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) dt = (dt && oldest_submit_jif != now && time_before(now, oldest_submit_jif + dt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) ? oldest_submit_jif + dt : now + et;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) nt = time_before(ent, dt) ? ent : dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) spin_unlock_irq(&device->resource->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) mod_timer(&device->request_timer, nt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags