^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * RDMA Network Block Driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/hdreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/scatterlist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/idr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "rnbd-clt.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) MODULE_DESCRIPTION("RDMA Network Block Device Client");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static int rnbd_client_major;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) static DEFINE_IDA(index_ida);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static DEFINE_MUTEX(ida_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static DEFINE_MUTEX(sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static LIST_HEAD(sess_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * Maximum number of partitions an instance can have.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * 6 bits = 64 minors = 63 partitions (one minor is used for the device itself)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define RNBD_PART_BITS 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static inline bool rnbd_clt_get_sess(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) return refcount_inc_not_zero(&sess->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static void free_sess(struct rnbd_clt_session *sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static void rnbd_clt_put_sess(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) if (refcount_dec_and_test(&sess->refcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) free_sess(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static void rnbd_clt_put_dev(struct rnbd_clt_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) if (!refcount_dec_and_test(&dev->refcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) mutex_lock(&ida_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) ida_simple_remove(&index_ida, dev->clt_device_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) mutex_unlock(&ida_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) kfree(dev->hw_queues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) kfree(dev->pathname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) rnbd_clt_put_sess(dev->sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) mutex_destroy(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) kfree(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static inline bool rnbd_clt_get_dev(struct rnbd_clt_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) return refcount_inc_not_zero(&dev->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static int rnbd_clt_set_dev_attr(struct rnbd_clt_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) const struct rnbd_msg_open_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct rnbd_clt_session *sess = dev->sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (!rsp->logical_block_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) dev->device_id = le32_to_cpu(rsp->device_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) dev->nsectors = le64_to_cpu(rsp->nsectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) dev->logical_block_size = le16_to_cpu(rsp->logical_block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) dev->physical_block_size = le16_to_cpu(rsp->physical_block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) dev->max_write_same_sectors = le32_to_cpu(rsp->max_write_same_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) dev->max_discard_sectors = le32_to_cpu(rsp->max_discard_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) dev->discard_granularity = le32_to_cpu(rsp->discard_granularity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) dev->discard_alignment = le32_to_cpu(rsp->discard_alignment);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) dev->secure_discard = le16_to_cpu(rsp->secure_discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) dev->rotational = rsp->rotational;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) dev->max_hw_sectors = sess->max_io_size / SECTOR_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) dev->max_segments = BMAX_SEGMENTS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static int rnbd_clt_change_capacity(struct rnbd_clt_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) size_t new_nsectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) rnbd_clt_info(dev, "Device size changed from %zu to %zu sectors\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) dev->nsectors, new_nsectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) dev->nsectors = new_nsectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) set_capacity(dev->gd, dev->nsectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) revalidate_disk_size(dev->gd, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static int process_msg_open_rsp(struct rnbd_clt_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) struct rnbd_msg_open_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) mutex_lock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (dev->dev_state == DEV_STATE_UNMAPPED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) rnbd_clt_info(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) "Ignoring Open-Response message from server for unmapped device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) err = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) u64 nsectors = le64_to_cpu(rsp->nsectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * If the device was remapped and the size changed in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * meantime we need to revalidate it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (dev->nsectors != nsectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) rnbd_clt_change_capacity(dev, nsectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) rnbd_clt_info(dev, "Device online, device remapped successfully\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) err = rnbd_clt_set_dev_attr(dev, rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) dev->dev_state = DEV_STATE_MAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) mutex_unlock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) int rnbd_clt_resize_disk(struct rnbd_clt_dev *dev, size_t newsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) mutex_lock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (dev->dev_state != DEV_STATE_MAPPED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) pr_err("Failed to set new size of the device, device is not opened\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) ret = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) ret = rnbd_clt_change_capacity(dev, newsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) mutex_unlock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static inline void rnbd_clt_dev_requeue(struct rnbd_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (WARN_ON(!q->hctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) /* We can come here from interrupt, thus async=true */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) blk_mq_run_hw_queue(q->hctx, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) RNBD_DELAY_IFBUSY = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * rnbd_get_cpu_qlist() - finds a list with HW queues to be rerun
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * @sess: Session to find a queue for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * @cpu: Cpu to start the search from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * Each CPU has a list of HW queues, which needs to be rerun. If a list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * is not empty - it is marked with a bit. This function finds first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * set bit in a bitmap and returns corresponding CPU list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static struct rnbd_cpu_qlist *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) rnbd_get_cpu_qlist(struct rnbd_clt_session *sess, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) int bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) /* Search from cpu to nr_cpu_ids */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) bit = find_next_bit(sess->cpu_queues_bm, nr_cpu_ids, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (bit < nr_cpu_ids) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return per_cpu_ptr(sess->cpu_queues, bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) } else if (cpu != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) /* Search from 0 to cpu */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) bit = find_next_bit(sess->cpu_queues_bm, cpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (bit < cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return per_cpu_ptr(sess->cpu_queues, bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) static inline int nxt_cpu(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return (cpu + 1) % nr_cpu_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * rnbd_rerun_if_needed() - rerun next queue marked as stopped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * @sess: Session to rerun a queue on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * Each CPU has it's own list of HW queues, which should be rerun.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * Function finds such list with HW queues, takes a list lock, picks up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * the first HW queue out of the list and requeues it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * Return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * True if the queue was requeued, false otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * Context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * Does not matter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static bool rnbd_rerun_if_needed(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct rnbd_queue *q = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) struct rnbd_cpu_qlist *cpu_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) int *cpup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * To keep fairness and not to let other queues starve we always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * try to wake up someone else in round-robin manner. That of course
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * increases latency but queues always have a chance to be executed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) cpup = get_cpu_ptr(sess->cpu_rr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) for (cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(*cpup)); cpu_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(cpu_q->cpu))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (!spin_trylock_irqsave(&cpu_q->requeue_lock, flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (unlikely(!test_bit(cpu_q->cpu, sess->cpu_queues_bm)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) q = list_first_entry_or_null(&cpu_q->requeue_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) typeof(*q), requeue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if (WARN_ON(!q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) goto clear_bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) list_del_init(&q->requeue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) clear_bit_unlock(0, &q->in_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (list_empty(&cpu_q->requeue_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /* Clear bit if nothing is left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) clear_bit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) clear_bit(cpu_q->cpu, sess->cpu_queues_bm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) spin_unlock_irqrestore(&cpu_q->requeue_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * Saves the CPU that is going to be requeued on the per-cpu var. Just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * incrementing it doesn't work because rnbd_get_cpu_qlist() will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * always return the first CPU with something on the queue list when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * value stored on the var is greater than the last CPU with something
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * on the list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (cpu_q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) *cpup = cpu_q->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) put_cpu_var(sess->cpu_rr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) if (q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) rnbd_clt_dev_requeue(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * rnbd_rerun_all_if_idle() - rerun all queues left in the list if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * session is idling (there are no requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * in-flight).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * @sess: Session to rerun the queues on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * This function tries to rerun all stopped queues if there are no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * requests in-flight anymore. This function tries to solve an obvious
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * problem, when number of tags < than number of queues (hctx), which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * are stopped and put to sleep. If last permit, which has been just put,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * does not wake up all left queues (hctxs), IO requests hang forever.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * That can happen when all number of permits, say N, have been exhausted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * from one CPU, and we have many block devices per session, say M.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * Each block device has it's own queue (hctx) for each CPU, so eventually
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * we can put that number of queues (hctxs) to sleep: M x nr_cpu_ids.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * If number of permits N < M x nr_cpu_ids finally we will get an IO hang.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * To avoid this hang last caller of rnbd_put_permit() (last caller is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * one who observes sess->busy == 0) must wake up all remaining queues.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * Context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * Does not matter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static void rnbd_rerun_all_if_idle(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) bool requeued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) requeued = rnbd_rerun_if_needed(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) } while (atomic_read(&sess->busy) == 0 && requeued);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static struct rtrs_permit *rnbd_get_permit(struct rnbd_clt_session *sess,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) enum rtrs_clt_con_type con_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) int wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct rtrs_permit *permit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) permit = rtrs_clt_get_permit(sess->rtrs, con_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) wait ? RTRS_PERMIT_WAIT :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) RTRS_PERMIT_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (likely(permit))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /* We have a subtle rare case here, when all permits can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * consumed before busy counter increased. This is safe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * because loser will get NULL as a permit, observe 0 busy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * counter and immediately restart the queue himself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) atomic_inc(&sess->busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return permit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) static void rnbd_put_permit(struct rnbd_clt_session *sess,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) struct rtrs_permit *permit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) rtrs_clt_put_permit(sess->rtrs, permit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) atomic_dec(&sess->busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) /* Paired with rnbd_clt_dev_add_to_requeue(). Decrement first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * and then check queue bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) rnbd_rerun_all_if_idle(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) static struct rnbd_iu *rnbd_get_iu(struct rnbd_clt_session *sess,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) enum rtrs_clt_con_type con_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) int wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) struct rnbd_iu *iu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) struct rtrs_permit *permit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) permit = rnbd_get_permit(sess, con_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) wait ? RTRS_PERMIT_WAIT :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) RTRS_PERMIT_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (unlikely(!permit))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) iu = rtrs_permit_to_pdu(permit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) iu->permit = permit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * 1st reference is dropped after finishing sending a "user" message,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * 2nd reference is dropped after confirmation with the response is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) * returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * 1st and 2nd can happen in any order, so the rnbd_iu should be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * released (rtrs_permit returned to ibbtrs) only leased after both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) * are finished.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) atomic_set(&iu->refcount, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) init_waitqueue_head(&iu->comp.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) iu->comp.errno = INT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) return iu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) static void rnbd_put_iu(struct rnbd_clt_session *sess, struct rnbd_iu *iu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (atomic_dec_and_test(&iu->refcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) rnbd_put_permit(sess, iu->permit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static void rnbd_softirq_done_fn(struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct rnbd_clt_dev *dev = rq->rq_disk->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) struct rnbd_clt_session *sess = dev->sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct rnbd_iu *iu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) iu = blk_mq_rq_to_pdu(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) rnbd_put_permit(sess, iu->permit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) blk_mq_end_request(rq, errno_to_blk_status(iu->errno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) static void msg_io_conf(void *priv, int errno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) struct rnbd_iu *iu = priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) struct rnbd_clt_dev *dev = iu->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) struct request *rq = iu->rq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) int rw = rq_data_dir(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) iu->errno = errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) blk_mq_complete_request(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (errno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) rnbd_clt_info_rl(dev, "%s I/O failed with err: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) rw == READ ? "read" : "write", errno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static void wake_up_iu_comp(struct rnbd_iu *iu, int errno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) iu->comp.errno = errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) wake_up(&iu->comp.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) static void msg_conf(void *priv, int errno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) struct rnbd_iu *iu = priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) iu->errno = errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) schedule_work(&iu->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) enum wait_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) NO_WAIT = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) WAIT = 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) static int send_usr_msg(struct rtrs_clt *rtrs, int dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) struct rnbd_iu *iu, struct kvec *vec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) size_t len, struct scatterlist *sg, unsigned int sg_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) void (*conf)(struct work_struct *work),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) int *errno, enum wait_type wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) struct rtrs_clt_req_ops req_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) INIT_WORK(&iu->work, conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) req_ops = (struct rtrs_clt_req_ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) .priv = iu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) .conf_fn = msg_conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) err = rtrs_clt_request(dir, &req_ops, rtrs, iu->permit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) vec, 1, len, sg, sg_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (!err && wait) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) wait_event(iu->comp.wait, iu->comp.errno != INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) *errno = iu->comp.errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) *errno = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) static void msg_close_conf(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) struct rnbd_clt_dev *dev = iu->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) wake_up_iu_comp(iu, iu->errno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) rnbd_put_iu(dev->sess, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) rnbd_clt_put_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) static int send_msg_close(struct rnbd_clt_dev *dev, u32 device_id, bool wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) struct rnbd_clt_session *sess = dev->sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) struct rnbd_msg_close msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) struct rnbd_iu *iu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) struct kvec vec = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) .iov_base = &msg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) .iov_len = sizeof(msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) int err, errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (!iu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) iu->buf = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) iu->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) sg_mark_end(&iu->sglist[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) msg.hdr.type = cpu_to_le16(RNBD_MSG_CLOSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) msg.device_id = cpu_to_le32(device_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) WARN_ON(!rnbd_clt_get_dev(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) err = send_usr_msg(sess->rtrs, WRITE, iu, &vec, 0, NULL, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) msg_close_conf, &errno, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) rnbd_clt_put_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) rnbd_put_iu(sess, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) err = errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) rnbd_put_iu(sess, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static void msg_open_conf(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) struct rnbd_msg_open_rsp *rsp = iu->buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) struct rnbd_clt_dev *dev = iu->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) int errno = iu->errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (errno) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) rnbd_clt_err(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) "Opening failed, server responded: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) errno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) errno = process_msg_open_rsp(dev, rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) if (errno) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) u32 device_id = le32_to_cpu(rsp->device_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * If server thinks its fine, but we fail to process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * then be nice and send a close to server.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) (void)send_msg_close(dev, device_id, NO_WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) kfree(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) wake_up_iu_comp(iu, errno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) rnbd_put_iu(dev->sess, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) rnbd_clt_put_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) static void msg_sess_info_conf(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) struct rnbd_msg_sess_info_rsp *rsp = iu->buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) struct rnbd_clt_session *sess = iu->sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (!iu->errno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) sess->ver = min_t(u8, rsp->ver, RNBD_PROTO_VER_MAJOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) kfree(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) wake_up_iu_comp(iu, iu->errno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) rnbd_put_iu(sess, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) rnbd_clt_put_sess(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) static int send_msg_open(struct rnbd_clt_dev *dev, bool wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct rnbd_clt_session *sess = dev->sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) struct rnbd_msg_open_rsp *rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct rnbd_msg_open msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) struct rnbd_iu *iu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) struct kvec vec = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) .iov_base = &msg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) .iov_len = sizeof(msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) int err, errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) if (!rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (!iu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) kfree(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) iu->buf = rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) iu->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) sg_init_one(iu->sglist, rsp, sizeof(*rsp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) msg.hdr.type = cpu_to_le16(RNBD_MSG_OPEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) msg.access_mode = dev->access_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) strlcpy(msg.dev_name, dev->pathname, sizeof(msg.dev_name));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) WARN_ON(!rnbd_clt_get_dev(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) err = send_usr_msg(sess->rtrs, READ, iu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) &vec, sizeof(*rsp), iu->sglist, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) msg_open_conf, &errno, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) rnbd_clt_put_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) rnbd_put_iu(sess, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) kfree(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) err = errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) rnbd_put_iu(sess, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) static int send_msg_sess_info(struct rnbd_clt_session *sess, bool wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) struct rnbd_msg_sess_info_rsp *rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) struct rnbd_msg_sess_info msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) struct rnbd_iu *iu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) struct kvec vec = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) .iov_base = &msg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) .iov_len = sizeof(msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) int err, errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (!rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) if (!iu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) kfree(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) iu->buf = rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) iu->sess = sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) sg_init_one(iu->sglist, rsp, sizeof(*rsp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) msg.hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) msg.ver = RNBD_PROTO_VER_MAJOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) if (!rnbd_clt_get_sess(sess)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * That can happen only in one case, when RTRS has restablished
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * the connection and link_ev() is called, but session is almost
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * dead, last reference on session is put and caller is waiting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * for RTRS to close everything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) goto put_iu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) err = send_usr_msg(sess->rtrs, READ, iu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) &vec, sizeof(*rsp), iu->sglist, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) msg_sess_info_conf, &errno, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) rnbd_clt_put_sess(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) put_iu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) rnbd_put_iu(sess, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) kfree(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) err = errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) rnbd_put_iu(sess, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) static void set_dev_states_to_disconnected(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) struct rnbd_clt_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) mutex_lock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) list_for_each_entry(dev, &sess->devs_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) rnbd_clt_err(dev, "Device disconnected.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) mutex_lock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (dev->dev_state == DEV_STATE_MAPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) dev->dev_state = DEV_STATE_MAPPED_DISCONNECTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) mutex_unlock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) mutex_unlock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) static void remap_devs(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) struct rnbd_clt_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) struct rtrs_attrs attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) * Careful here: we are called from RTRS link event directly,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * thus we can't send any RTRS request and wait for response
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * or RTRS will not be able to complete request with failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * if something goes wrong (failing of outstanding requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * happens exactly from the context where we are blocking now).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * So to avoid deadlocks each usr message sent from here must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * be asynchronous.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) err = send_msg_sess_info(sess, NO_WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) pr_err("send_msg_sess_info(\"%s\"): %d\n", sess->sessname, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) err = rtrs_clt_query(sess->rtrs, &attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) pr_err("rtrs_clt_query(\"%s\"): %d\n", sess->sessname, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) mutex_lock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) sess->max_io_size = attrs.max_io_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) list_for_each_entry(dev, &sess->devs_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) bool skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) mutex_lock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) skip = (dev->dev_state == DEV_STATE_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) mutex_unlock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) if (skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * When device is establishing connection for the first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * time - do not remap, it will be closed soon.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) rnbd_clt_info(dev, "session reconnected, remapping device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) err = send_msg_open(dev, NO_WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) rnbd_clt_err(dev, "send_msg_open(): %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) mutex_unlock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) static void rnbd_clt_link_ev(void *priv, enum rtrs_clt_link_ev ev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) struct rnbd_clt_session *sess = priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) switch (ev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) case RTRS_CLT_LINK_EV_DISCONNECTED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) set_dev_states_to_disconnected(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) case RTRS_CLT_LINK_EV_RECONNECTED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) remap_devs(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) pr_err("Unknown session event received (%d), session: %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) ev, sess->sessname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) static void rnbd_init_cpu_qlists(struct rnbd_cpu_qlist __percpu *cpu_queues)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) struct rnbd_cpu_qlist *cpu_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) cpu_q = per_cpu_ptr(cpu_queues, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) cpu_q->cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) INIT_LIST_HEAD(&cpu_q->requeue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) spin_lock_init(&cpu_q->requeue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) static void destroy_mq_tags(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (sess->tag_set.tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) blk_mq_free_tag_set(&sess->tag_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) static inline void wake_up_rtrs_waiters(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) sess->rtrs_ready = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) wake_up_all(&sess->rtrs_waitq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) static void close_rtrs(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) if (!IS_ERR_OR_NULL(sess->rtrs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) rtrs_clt_close(sess->rtrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) sess->rtrs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) wake_up_rtrs_waiters(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) static void free_sess(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) WARN_ON(!list_empty(&sess->devs_list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) close_rtrs(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) destroy_mq_tags(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) if (!list_empty(&sess->list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) mutex_lock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) list_del(&sess->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) mutex_unlock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) free_percpu(sess->cpu_queues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) free_percpu(sess->cpu_rr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) mutex_destroy(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) kfree(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) static struct rnbd_clt_session *alloc_sess(const char *sessname)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) struct rnbd_clt_session *sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) int err, cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) sess = kzalloc_node(sizeof(*sess), GFP_KERNEL, NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) if (!sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) strlcpy(sess->sessname, sessname, sizeof(sess->sessname));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) atomic_set(&sess->busy, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) mutex_init(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) INIT_LIST_HEAD(&sess->devs_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) INIT_LIST_HEAD(&sess->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) bitmap_zero(sess->cpu_queues_bm, NR_CPUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) init_waitqueue_head(&sess->rtrs_waitq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) refcount_set(&sess->refcount, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) sess->cpu_queues = alloc_percpu(struct rnbd_cpu_qlist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) if (!sess->cpu_queues) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) rnbd_init_cpu_qlists(sess->cpu_queues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * That is simple percpu variable which stores cpu indeces, which are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) * incremented on each access. We need that for the sake of fairness
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * to wake up queues in a round-robin manner.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) sess->cpu_rr = alloc_percpu(int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) if (!sess->cpu_rr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) * per_cpu_ptr(sess->cpu_rr, cpu) = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) return sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) free_sess(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) return ERR_PTR(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) static int wait_for_rtrs_connection(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) wait_event(sess->rtrs_waitq, sess->rtrs_ready);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) if (IS_ERR_OR_NULL(sess->rtrs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) return -ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) static void wait_for_rtrs_disconnection(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) __releases(&sess_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) __acquires(&sess_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) DEFINE_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) prepare_to_wait(&sess->rtrs_waitq, &wait, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (IS_ERR_OR_NULL(sess->rtrs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) finish_wait(&sess->rtrs_waitq, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) mutex_unlock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) /* loop in caller, see __find_and_get_sess().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) * You can't leave mutex locked and call schedule(), you will catch a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) * deadlock with a caller of free_sess(), which has just put the last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) * reference and is about to take the sess_lock in order to delete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) * the session from the list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) mutex_lock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) static struct rnbd_clt_session *__find_and_get_sess(const char *sessname)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) __releases(&sess_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) __acquires(&sess_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) struct rnbd_clt_session *sess, *sn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) list_for_each_entry_safe(sess, sn, &sess_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) if (strcmp(sessname, sess->sessname))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) if (sess->rtrs_ready && IS_ERR_OR_NULL(sess->rtrs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * No RTRS connection, session is dying.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) if (rnbd_clt_get_sess(sess)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * Alive session is found, wait for RTRS connection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) mutex_unlock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) err = wait_for_rtrs_connection(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) rnbd_clt_put_sess(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) mutex_lock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) /* Session is dying, repeat the loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) return sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * Ref is 0, session is dying, wait for RTRS disconnect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * in order to avoid session names clashes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) wait_for_rtrs_disconnection(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) * RTRS is disconnected and soon session will be freed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) * so repeat a loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) static struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) rnbd_clt_session *find_or_create_sess(const char *sessname, bool *first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) struct rnbd_clt_session *sess = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) mutex_lock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) sess = __find_and_get_sess(sessname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) if (!sess) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) sess = alloc_sess(sessname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (IS_ERR(sess)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) mutex_unlock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) return sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) list_add(&sess->list, &sess_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) *first = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) *first = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) mutex_unlock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) return sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) static int rnbd_client_open(struct block_device *block_device, fmode_t mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) struct rnbd_clt_dev *dev = block_device->bd_disk->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) if (dev->read_only && (mode & FMODE_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) if (dev->dev_state == DEV_STATE_UNMAPPED ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) !rnbd_clt_get_dev(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) static void rnbd_client_release(struct gendisk *gen, fmode_t mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) struct rnbd_clt_dev *dev = gen->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) rnbd_clt_put_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) static int rnbd_client_getgeo(struct block_device *block_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) struct hd_geometry *geo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) u64 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) struct rnbd_clt_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) dev = block_device->bd_disk->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) size = dev->size * (dev->logical_block_size / SECTOR_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) geo->cylinders = size >> 6; /* size/64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) geo->heads = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) geo->sectors = 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) geo->start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) static const struct block_device_operations rnbd_client_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) .open = rnbd_client_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) .release = rnbd_client_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) .getgeo = rnbd_client_getgeo
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) /* The amount of data that belongs to an I/O and the amount of data that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) * should be read or written to the disk (bi_size) can differ.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) * E.g. When WRITE_SAME is used, only a small amount of data is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) * transferred that is then written repeatedly over a lot of sectors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) * Get the size of data to be transferred via RTRS by summing up the size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) * of the scather-gather list entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) static size_t rnbd_clt_get_sg_size(struct scatterlist *sglist, u32 len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) size_t tsize = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) for_each_sg(sglist, sg, len, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) tsize += sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) return tsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) struct request *rq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) struct rnbd_iu *iu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) struct rtrs_clt *rtrs = dev->sess->rtrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) struct rtrs_permit *permit = iu->permit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) struct rnbd_msg_io msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) struct rtrs_clt_req_ops req_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) unsigned int sg_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) struct kvec vec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) iu->rq = rq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) iu->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) msg.sector = cpu_to_le64(blk_rq_pos(rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) msg.bi_size = cpu_to_le32(blk_rq_bytes(rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) msg.rw = cpu_to_le32(rq_to_rnbd_flags(rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) msg.prio = cpu_to_le16(req_get_ioprio(rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) * We only support discards with single segment for now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) * See queue limits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) if (req_op(rq) != REQ_OP_DISCARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) sg_cnt = blk_rq_map_sg(dev->queue, rq, iu->sglist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) if (sg_cnt == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) /* Do not forget to mark the end */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) sg_mark_end(&iu->sglist[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) msg.hdr.type = cpu_to_le16(RNBD_MSG_IO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) msg.device_id = cpu_to_le32(dev->device_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) vec = (struct kvec) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) .iov_base = &msg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) .iov_len = sizeof(msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) size = rnbd_clt_get_sg_size(iu->sglist, sg_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) req_ops = (struct rtrs_clt_req_ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) .priv = iu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) .conf_fn = msg_io_conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) err = rtrs_clt_request(rq_data_dir(rq), &req_ops, rtrs, permit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) &vec, 1, size, iu->sglist, sg_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) if (unlikely(err)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) rnbd_clt_err_rl(dev, "RTRS failed to transfer IO, err: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) * rnbd_clt_dev_add_to_requeue() - add device to requeue if session is busy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) * @dev: Device to be checked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) * @q: Queue to be added to the requeue list if required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) * If session is busy, that means someone will requeue us when resources
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) * are freed. If session is not doing anything - device is not added to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) * the list and @false is returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) static bool rnbd_clt_dev_add_to_requeue(struct rnbd_clt_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) struct rnbd_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) struct rnbd_clt_session *sess = dev->sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) struct rnbd_cpu_qlist *cpu_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) bool added = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) bool need_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) cpu_q = get_cpu_ptr(sess->cpu_queues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) spin_lock_irqsave(&cpu_q->requeue_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) if (likely(!test_and_set_bit_lock(0, &q->in_list))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) if (WARN_ON(!list_empty(&q->requeue_list)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) need_set = !test_bit(cpu_q->cpu, sess->cpu_queues_bm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) if (need_set) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) set_bit(cpu_q->cpu, sess->cpu_queues_bm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) /* Paired with rnbd_put_permit(). Set a bit first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) * and then observe the busy counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) if (likely(atomic_read(&sess->busy))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) list_add_tail(&q->requeue_list, &cpu_q->requeue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) /* Very unlikely, but possible: busy counter was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * observed as zero. Drop all bits and return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) * false to restart the queue by ourselves.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) if (need_set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) clear_bit(cpu_q->cpu, sess->cpu_queues_bm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) clear_bit_unlock(0, &q->in_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) added = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) spin_unlock_irqrestore(&cpu_q->requeue_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) put_cpu_ptr(sess->cpu_queues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) return added;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) static void rnbd_clt_dev_kick_mq_queue(struct rnbd_clt_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) struct blk_mq_hw_ctx *hctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) int delay)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) struct rnbd_queue *q = hctx->driver_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) if (delay != RNBD_DELAY_IFBUSY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) blk_mq_delay_run_hw_queue(hctx, delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) else if (unlikely(!rnbd_clt_dev_add_to_requeue(dev, q)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) * If session is not busy we have to restart
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) * the queue ourselves.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) blk_mq_delay_run_hw_queue(hctx, 10/*ms*/);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) const struct blk_mq_queue_data *bd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) struct request *rq = bd->rq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) struct rnbd_clt_dev *dev = rq->rq_disk->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) if (unlikely(dev->dev_state != DEV_STATE_MAPPED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) iu->permit = rnbd_get_permit(dev->sess, RTRS_IO_CON,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) RTRS_PERMIT_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) if (unlikely(!iu->permit)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) rnbd_clt_dev_kick_mq_queue(dev, hctx, RNBD_DELAY_IFBUSY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) return BLK_STS_RESOURCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) blk_mq_start_request(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) err = rnbd_client_xfer_request(dev, rq, iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) if (likely(err == 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) return BLK_STS_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) if (unlikely(err == -EAGAIN || err == -ENOMEM)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) rnbd_put_permit(dev->sess, iu->permit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) return BLK_STS_RESOURCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) rnbd_put_permit(dev->sess, iu->permit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) static int rnbd_init_request(struct blk_mq_tag_set *set, struct request *rq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) unsigned int hctx_idx, unsigned int numa_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) sg_init_table(iu->sglist, BMAX_SEGMENTS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) static struct blk_mq_ops rnbd_mq_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) .queue_rq = rnbd_queue_rq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) .init_request = rnbd_init_request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) .complete = rnbd_softirq_done_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) static int setup_mq_tags(struct rnbd_clt_session *sess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) struct blk_mq_tag_set *tag_set = &sess->tag_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) memset(tag_set, 0, sizeof(*tag_set));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) tag_set->ops = &rnbd_mq_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) tag_set->queue_depth = sess->queue_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) tag_set->numa_node = NUMA_NO_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) tag_set->flags = BLK_MQ_F_SHOULD_MERGE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) BLK_MQ_F_TAG_QUEUE_SHARED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) tag_set->cmd_size = sizeof(struct rnbd_iu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) tag_set->nr_hw_queues = num_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) return blk_mq_alloc_tag_set(tag_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) static struct rnbd_clt_session *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) find_and_get_or_create_sess(const char *sessname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) const struct rtrs_addr *paths,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) size_t path_cnt, u16 port_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) struct rnbd_clt_session *sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) struct rtrs_attrs attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) bool first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) struct rtrs_clt_ops rtrs_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) sess = find_or_create_sess(sessname, &first);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) if (sess == ERR_PTR(-ENOMEM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) else if (!first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) return sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) rtrs_ops = (struct rtrs_clt_ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) .priv = sess,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) .link_ev = rnbd_clt_link_ev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * Nothing was found, establish rtrs connection and proceed further.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) sess->rtrs = rtrs_clt_open(&rtrs_ops, sessname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) paths, path_cnt, port_nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) sizeof(struct rnbd_iu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) RECONNECT_DELAY, BMAX_SEGMENTS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) BLK_MAX_SEGMENT_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) MAX_RECONNECTS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) if (IS_ERR(sess->rtrs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) err = PTR_ERR(sess->rtrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) goto wake_up_and_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) err = rtrs_clt_query(sess->rtrs, &attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) goto close_rtrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) sess->max_io_size = attrs.max_io_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) sess->queue_depth = attrs.queue_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) err = setup_mq_tags(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) goto close_rtrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) err = send_msg_sess_info(sess, WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) goto close_rtrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) wake_up_rtrs_waiters(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) return sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) close_rtrs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) close_rtrs(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) put_sess:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) rnbd_clt_put_sess(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) return ERR_PTR(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) wake_up_and_put:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) wake_up_rtrs_waiters(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) goto put_sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) static inline void rnbd_init_hw_queue(struct rnbd_clt_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) struct rnbd_queue *q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) struct blk_mq_hw_ctx *hctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) INIT_LIST_HEAD(&q->requeue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) q->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) q->hctx = hctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) static void rnbd_init_mq_hw_queues(struct rnbd_clt_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) struct blk_mq_hw_ctx *hctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) struct rnbd_queue *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) queue_for_each_hw_ctx(dev->queue, hctx, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) q = &dev->hw_queues[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) rnbd_init_hw_queue(dev, q, hctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) hctx->driver_data = q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) static int setup_mq_dev(struct rnbd_clt_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) dev->queue = blk_mq_init_queue(&dev->sess->tag_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) if (IS_ERR(dev->queue)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) rnbd_clt_err(dev, "Initializing multiqueue queue failed, err: %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) PTR_ERR(dev->queue));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) return PTR_ERR(dev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) rnbd_init_mq_hw_queues(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) static void setup_request_queue(struct rnbd_clt_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) blk_queue_logical_block_size(dev->queue, dev->logical_block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) blk_queue_physical_block_size(dev->queue, dev->physical_block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) blk_queue_max_hw_sectors(dev->queue, dev->max_hw_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) blk_queue_max_write_same_sectors(dev->queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) dev->max_write_same_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) * we don't support discards to "discontiguous" segments
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) * in on request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) blk_queue_max_discard_segments(dev->queue, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) blk_queue_max_discard_sectors(dev->queue, dev->max_discard_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) dev->queue->limits.discard_granularity = dev->discard_granularity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) dev->queue->limits.discard_alignment = dev->discard_alignment;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) if (dev->max_discard_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) blk_queue_flag_set(QUEUE_FLAG_DISCARD, dev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) if (dev->secure_discard)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) blk_queue_flag_set(QUEUE_FLAG_SECERASE, dev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, dev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, dev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) blk_queue_max_segments(dev->queue, dev->max_segments);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) blk_queue_io_opt(dev->queue, dev->sess->max_io_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) blk_queue_virt_boundary(dev->queue, SZ_4K - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) blk_queue_write_cache(dev->queue, true, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) dev->queue->queuedata = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) static void rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) dev->gd->major = rnbd_client_major;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) dev->gd->first_minor = idx << RNBD_PART_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) dev->gd->fops = &rnbd_client_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) dev->gd->queue = dev->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) dev->gd->private_data = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) snprintf(dev->gd->disk_name, sizeof(dev->gd->disk_name), "rnbd%d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) pr_debug("disk_name=%s, capacity=%zu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) dev->gd->disk_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) dev->nsectors * (dev->logical_block_size / SECTOR_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) set_capacity(dev->gd, dev->nsectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) if (dev->access_mode == RNBD_ACCESS_RO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) dev->read_only = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) set_disk_ro(dev->gd, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) dev->read_only = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) if (!dev->rotational)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) blk_queue_flag_set(QUEUE_FLAG_NONROT, dev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) static int rnbd_client_setup_device(struct rnbd_clt_session *sess,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) struct rnbd_clt_dev *dev, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) dev->size = dev->nsectors * dev->logical_block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) err = setup_mq_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) setup_request_queue(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) dev->gd = alloc_disk_node(1 << RNBD_PART_BITS, NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) if (!dev->gd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) blk_cleanup_queue(dev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) rnbd_clt_setup_gen_disk(dev, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) enum rnbd_access_mode access_mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) const char *pathname)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) struct rnbd_clt_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) if (!dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) dev->hw_queues = kcalloc(nr_cpu_ids, sizeof(*dev->hw_queues),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) if (!dev->hw_queues) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) goto out_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) mutex_lock(&ida_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) ret = ida_simple_get(&index_ida, 0, 1 << (MINORBITS - RNBD_PART_BITS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) mutex_unlock(&ida_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) pr_err("Failed to initialize device '%s' from session %s, allocating idr failed, err: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) pathname, sess->sessname, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) goto out_queues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) dev->pathname = kstrdup(pathname, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) if (!dev->pathname) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) goto out_queues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) dev->clt_device_id = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) dev->sess = sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) dev->access_mode = access_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) mutex_init(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) refcount_set(&dev->refcount, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) dev->dev_state = DEV_STATE_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) * Here we called from sysfs entry, thus clt-sysfs is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) * responsible that session will not disappear.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) WARN_ON(!rnbd_clt_get_sess(sess));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) return dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) out_queues:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) kfree(dev->hw_queues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) out_alloc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) kfree(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) static bool __exists_dev(const char *pathname)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) struct rnbd_clt_session *sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) struct rnbd_clt_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) bool found = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) list_for_each_entry(sess, &sess_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) mutex_lock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) list_for_each_entry(dev, &sess->devs_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) if (strlen(dev->pathname) == strlen(pathname) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) !strcmp(dev->pathname, pathname)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) found = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) mutex_unlock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) if (found)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) static bool exists_devpath(const char *pathname)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) bool found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) mutex_lock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) found = __exists_dev(pathname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) mutex_unlock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) static bool insert_dev_if_not_exists_devpath(const char *pathname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) struct rnbd_clt_session *sess,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) struct rnbd_clt_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) bool found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) mutex_lock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) found = __exists_dev(pathname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) if (!found) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) mutex_lock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) list_add_tail(&dev->list, &sess->devs_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) mutex_unlock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) mutex_unlock(&sess_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) static void delete_dev(struct rnbd_clt_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) struct rnbd_clt_session *sess = dev->sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) mutex_lock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) list_del(&dev->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) mutex_unlock(&sess->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) struct rnbd_clt_dev *rnbd_clt_map_device(const char *sessname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) struct rtrs_addr *paths,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) size_t path_cnt, u16 port_nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) const char *pathname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) enum rnbd_access_mode access_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) struct rnbd_clt_session *sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) struct rnbd_clt_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) if (exists_devpath(pathname))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) return ERR_PTR(-EEXIST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) sess = find_and_get_or_create_sess(sessname, paths, path_cnt, port_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) if (IS_ERR(sess))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) return ERR_CAST(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) dev = init_dev(sess, access_mode, pathname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) if (IS_ERR(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) pr_err("map_device: failed to map device '%s' from session %s, can't initialize device, err: %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) pathname, sess->sessname, PTR_ERR(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) ret = PTR_ERR(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) goto put_sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) if (insert_dev_if_not_exists_devpath(pathname, sess, dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) ret = -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) goto put_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) ret = send_msg_open(dev, WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) rnbd_clt_err(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) "map_device: failed, can't open remote device, err: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) goto del_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) mutex_lock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) pr_debug("Opened remote device: session=%s, path='%s'\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) sess->sessname, pathname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) ret = rnbd_client_setup_device(sess, dev, dev->clt_device_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) rnbd_clt_err(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) "map_device: Failed to configure device, err: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) mutex_unlock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) goto send_close;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) rnbd_clt_info(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) "map_device: Device mapped as %s (nsectors: %zu, logical_block_size: %d, physical_block_size: %d, max_write_same_sectors: %d, max_discard_sectors: %d, discard_granularity: %d, discard_alignment: %d, secure_discard: %d, max_segments: %d, max_hw_sectors: %d, rotational: %d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) dev->gd->disk_name, dev->nsectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) dev->logical_block_size, dev->physical_block_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) dev->max_write_same_sectors, dev->max_discard_sectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) dev->discard_granularity, dev->discard_alignment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) dev->secure_discard, dev->max_segments,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) dev->max_hw_sectors, dev->rotational);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) mutex_unlock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) add_disk(dev->gd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) rnbd_clt_put_sess(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) return dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) send_close:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) send_msg_close(dev, dev->device_id, WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) del_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) delete_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) put_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) rnbd_clt_put_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) put_sess:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) rnbd_clt_put_sess(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) static void destroy_gen_disk(struct rnbd_clt_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) del_gendisk(dev->gd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) blk_cleanup_queue(dev->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) put_disk(dev->gd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) static void destroy_sysfs(struct rnbd_clt_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) const struct attribute *sysfs_self)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) rnbd_clt_remove_dev_symlink(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) if (dev->kobj.state_initialized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) if (sysfs_self)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) /* To avoid deadlock firstly remove itself */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) sysfs_remove_file_self(&dev->kobj, sysfs_self);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) kobject_del(&dev->kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) kobject_put(&dev->kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) int rnbd_clt_unmap_device(struct rnbd_clt_dev *dev, bool force,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) const struct attribute *sysfs_self)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) struct rnbd_clt_session *sess = dev->sess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) int refcount, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) bool was_mapped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) mutex_lock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) if (dev->dev_state == DEV_STATE_UNMAPPED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) rnbd_clt_info(dev, "Device is already being unmapped\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) ret = -EALREADY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) refcount = refcount_read(&dev->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) if (!force && refcount > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) rnbd_clt_err(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) "Closing device failed, device is in use, (%d device users)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) refcount - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) was_mapped = (dev->dev_state == DEV_STATE_MAPPED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) dev->dev_state = DEV_STATE_UNMAPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) mutex_unlock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) delete_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) destroy_sysfs(dev, sysfs_self);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) destroy_gen_disk(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) if (was_mapped && sess->rtrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) send_msg_close(dev, dev->device_id, WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) rnbd_clt_info(dev, "Device is unmapped\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) /* Likely last reference put */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) rnbd_clt_put_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) * Here device and session can be vanished!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) mutex_unlock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) int rnbd_clt_remap_device(struct rnbd_clt_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) mutex_lock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) else if (dev->dev_state == DEV_STATE_UNMAPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) err = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) else if (dev->dev_state == DEV_STATE_MAPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) err = -EALREADY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) mutex_unlock(&dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) rnbd_clt_info(dev, "Remapping device.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) err = send_msg_open(dev, WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) rnbd_clt_err(dev, "remap_device: %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) static void unmap_device_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) struct rnbd_clt_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) dev = container_of(work, typeof(*dev), unmap_on_rmmod_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) rnbd_clt_unmap_device(dev, true, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) static void rnbd_destroy_sessions(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) struct rnbd_clt_session *sess, *sn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) struct rnbd_clt_dev *dev, *tn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) /* Firstly forbid access through sysfs interface */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) rnbd_clt_destroy_default_group();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) rnbd_clt_destroy_sysfs_files();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) * Here at this point there is no any concurrent access to sessions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) * list and devices list:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) * 1. New session or device can'be be created - session sysfs files
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) * are removed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) * 2. Device or session can't be removed - module reference is taken
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) * into account in unmap device sysfs callback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) * 3. No IO requests inflight - each file open of block_dev increases
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) * module reference in get_disk().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) * But still there can be user requests inflights, which are sent by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) * asynchronous send_msg_*() functions, thus before unmapping devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) * RTRS session must be explicitly closed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) list_for_each_entry_safe(sess, sn, &sess_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) if (!rnbd_clt_get_sess(sess))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) close_rtrs(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) list_for_each_entry_safe(dev, tn, &sess->devs_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) * Here unmap happens in parallel for only one reason:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) * blk_cleanup_queue() takes around half a second, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) * on huge amount of devices the whole module unload
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) * procedure takes minutes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) INIT_WORK(&dev->unmap_on_rmmod_work, unmap_device_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) queue_work(system_long_wq, &dev->unmap_on_rmmod_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) rnbd_clt_put_sess(sess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) /* Wait for all scheduled unmap works */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) flush_workqueue(system_long_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) WARN_ON(!list_empty(&sess_list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) static int __init rnbd_client_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) BUILD_BUG_ON(sizeof(struct rnbd_msg_hdr) != 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info) != 36);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info_rsp) != 36);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) BUILD_BUG_ON(sizeof(struct rnbd_msg_open) != 264);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) BUILD_BUG_ON(sizeof(struct rnbd_msg_close) != 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) BUILD_BUG_ON(sizeof(struct rnbd_msg_open_rsp) != 56);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) rnbd_client_major = register_blkdev(rnbd_client_major, "rnbd");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) if (rnbd_client_major <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) pr_err("Failed to load module, block device registration failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) err = rnbd_clt_create_sysfs_files();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) pr_err("Failed to load module, creating sysfs device files failed, err: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) unregister_blkdev(rnbd_client_major, "rnbd");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) static void __exit rnbd_client_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) rnbd_destroy_sessions();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) unregister_blkdev(rnbd_client_major, "rnbd");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) ida_destroy(&index_ida);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) module_init(rnbd_client_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) module_exit(rnbd_client_exit);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags