^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * NVMe over Fabrics RDMA target.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (c) 2015-2016 HGST, a Western Digital Company.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/ctype.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/nvme.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/inet.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/unaligned.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <rdma/ib_verbs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <rdma/rdma_cm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <rdma/rw.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <rdma/ib_cm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/nvme-rdma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include "nvmet.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * We allow at least 1 page, up to 4 SGEs, and up to 16KB of inline data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define NVMET_RDMA_MAX_INLINE_SGE 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define NVMET_RDMA_MAX_INLINE_DATA_SIZE max_t(int, SZ_16K, PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /* Assume mpsmin == device_page_size == 4KB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define NVMET_RDMA_MAX_MDTS 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define NVMET_RDMA_MAX_METADATA_MDTS 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct nvmet_rdma_srq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) struct nvmet_rdma_cmd {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct ib_sge sge[NVMET_RDMA_MAX_INLINE_SGE + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct ib_cqe cqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct ib_recv_wr wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct scatterlist inline_sg[NVMET_RDMA_MAX_INLINE_SGE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) struct nvme_command *nvme_cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct nvmet_rdma_queue *queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct nvmet_rdma_srq *nsrq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) NVMET_RDMA_REQ_INLINE_DATA = (1 << 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) NVMET_RDMA_REQ_INVALIDATE_RKEY = (1 << 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct nvmet_rdma_rsp {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct ib_sge send_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) struct ib_cqe send_cqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) struct ib_send_wr send_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct nvmet_rdma_cmd *cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct nvmet_rdma_queue *queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct ib_cqe read_cqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct ib_cqe write_cqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct rdma_rw_ctx rw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct nvmet_req req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) bool allocated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) u8 n_rdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) u32 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) u32 invalidate_rkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct list_head wait_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct list_head free_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) enum nvmet_rdma_queue_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) NVMET_RDMA_Q_CONNECTING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) NVMET_RDMA_Q_LIVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) NVMET_RDMA_Q_DISCONNECTING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct nvmet_rdma_queue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) struct rdma_cm_id *cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct ib_qp *qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct nvmet_port *port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct ib_cq *cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) atomic_t sq_wr_avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct nvmet_rdma_device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct nvmet_rdma_srq *nsrq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) spinlock_t state_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) enum nvmet_rdma_queue_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct nvmet_cq nvme_cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) struct nvmet_sq nvme_sq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct nvmet_rdma_rsp *rsps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) struct list_head free_rsps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) spinlock_t rsps_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct nvmet_rdma_cmd *cmds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) struct work_struct release_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct list_head rsp_wait_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct list_head rsp_wr_wait_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) spinlock_t rsp_wr_wait_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) int host_qid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) int comp_vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) int recv_queue_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) int send_queue_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) struct list_head queue_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct nvmet_rdma_port {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct nvmet_port *nport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) struct sockaddr_storage addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) struct rdma_cm_id *cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) struct delayed_work repair_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct nvmet_rdma_srq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct ib_srq *srq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct nvmet_rdma_cmd *cmds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) struct nvmet_rdma_device *ndev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) struct nvmet_rdma_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) struct ib_device *device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct ib_pd *pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct nvmet_rdma_srq **srqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) int srq_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) size_t srq_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct kref ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct list_head entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) int inline_data_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) int inline_page_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static bool nvmet_rdma_use_srq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) module_param_named(use_srq, nvmet_rdma_use_srq, bool, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) MODULE_PARM_DESC(use_srq, "Use shared receive queue.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static int srq_size_set(const char *val, const struct kernel_param *kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) static const struct kernel_param_ops srq_size_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) .set = srq_size_set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) .get = param_get_int,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) static int nvmet_rdma_srq_size = 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) module_param_cb(srq_size, &srq_size_ops, &nvmet_rdma_srq_size, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) MODULE_PARM_DESC(srq_size, "set Shared Receive Queue (SRQ) size, should >= 256 (default: 1024)");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static DEFINE_IDA(nvmet_rdma_queue_ida);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static LIST_HEAD(nvmet_rdma_queue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) static DEFINE_MUTEX(nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) static LIST_HEAD(device_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static DEFINE_MUTEX(device_list_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static void nvmet_rdma_qp_event(struct ib_event *event, void *priv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) struct nvmet_rdma_rsp *r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) struct nvmet_rdma_rsp *r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static const struct nvmet_fabrics_ops nvmet_rdma_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) static int srq_size_set(const char *val, const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) int n = 0, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) ret = kstrtoint(val, 10, &n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (ret != 0 || n < 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return param_set_int(val, kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static int num_pages(int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) return 1 + (((len - 1) & PAGE_MASK) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) static inline bool nvmet_rdma_need_data_in(struct nvmet_rdma_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) return nvme_is_write(rsp->req.cmd) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) rsp->req.transfer_len &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static inline bool nvmet_rdma_need_data_out(struct nvmet_rdma_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) return !nvme_is_write(rsp->req.cmd) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) rsp->req.transfer_len &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) !rsp->req.cqe->status &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static inline struct nvmet_rdma_rsp *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) nvmet_rdma_get_rsp(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) struct nvmet_rdma_rsp *rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) spin_lock_irqsave(&queue->rsps_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) rsp = list_first_entry_or_null(&queue->free_rsps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) struct nvmet_rdma_rsp, free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (likely(rsp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) list_del(&rsp->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) spin_unlock_irqrestore(&queue->rsps_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (unlikely(!rsp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (unlikely(!rsp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) ret = nvmet_rdma_alloc_rsp(queue->dev, rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (unlikely(ret)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) kfree(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) rsp->allocated = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) nvmet_rdma_put_rsp(struct nvmet_rdma_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if (unlikely(rsp->allocated)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) nvmet_rdma_free_rsp(rsp->queue->dev, rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) kfree(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) spin_lock_irqsave(&rsp->queue->rsps_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) list_add_tail(&rsp->free_list, &rsp->queue->free_rsps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) spin_unlock_irqrestore(&rsp->queue->rsps_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) static void nvmet_rdma_free_inline_pages(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) struct nvmet_rdma_cmd *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct ib_sge *sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (!ndev->inline_data_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) sg = c->inline_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) sge = &c->sge[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) if (sge->length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) ib_dma_unmap_page(ndev->device, sge->addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) sge->length, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) if (sg_page(sg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) __free_page(sg_page(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) static int nvmet_rdma_alloc_inline_pages(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct nvmet_rdma_cmd *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) struct ib_sge *sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) struct page *pg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (!ndev->inline_data_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) sg = c->inline_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) sg_init_table(sg, ndev->inline_page_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) sge = &c->sge[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) len = ndev->inline_data_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) pg = alloc_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (!pg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) sg_assign_page(sg, pg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) sge->addr = ib_dma_map_page(ndev->device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) pg, 0, PAGE_SIZE, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (ib_dma_mapping_error(ndev->device, sge->addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) sge->length = min_t(int, len, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) sge->lkey = ndev->pd->local_dma_lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) len -= sge->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) out_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) for (; i >= 0; i--, sg--, sge--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (sge->length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) ib_dma_unmap_page(ndev->device, sge->addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) sge->length, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (sg_page(sg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) __free_page(sg_page(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) static int nvmet_rdma_alloc_cmd(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) struct nvmet_rdma_cmd *c, bool admin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /* NVMe command / RDMA RECV */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) c->nvme_cmd = kmalloc(sizeof(*c->nvme_cmd), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (!c->nvme_cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) c->sge[0].addr = ib_dma_map_single(ndev->device, c->nvme_cmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) sizeof(*c->nvme_cmd), DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (ib_dma_mapping_error(ndev->device, c->sge[0].addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) goto out_free_cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) c->sge[0].length = sizeof(*c->nvme_cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) c->sge[0].lkey = ndev->pd->local_dma_lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (!admin && nvmet_rdma_alloc_inline_pages(ndev, c))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) goto out_unmap_cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) c->cqe.done = nvmet_rdma_recv_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) c->wr.wr_cqe = &c->cqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) c->wr.sg_list = c->sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) c->wr.num_sge = admin ? 1 : ndev->inline_page_count + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) out_unmap_cmd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) ib_dma_unmap_single(ndev->device, c->sge[0].addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) sizeof(*c->nvme_cmd), DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) out_free_cmd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) kfree(c->nvme_cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) static void nvmet_rdma_free_cmd(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) struct nvmet_rdma_cmd *c, bool admin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) if (!admin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) nvmet_rdma_free_inline_pages(ndev, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) ib_dma_unmap_single(ndev->device, c->sge[0].addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) sizeof(*c->nvme_cmd), DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) kfree(c->nvme_cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) static struct nvmet_rdma_cmd *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) nvmet_rdma_alloc_cmds(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) int nr_cmds, bool admin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) struct nvmet_rdma_cmd *cmds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) int ret = -EINVAL, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) cmds = kcalloc(nr_cmds, sizeof(struct nvmet_rdma_cmd), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (!cmds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) for (i = 0; i < nr_cmds; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) ret = nvmet_rdma_alloc_cmd(ndev, cmds + i, admin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) return cmds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) while (--i >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) nvmet_rdma_free_cmd(ndev, cmds + i, admin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) kfree(cmds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) static void nvmet_rdma_free_cmds(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) struct nvmet_rdma_cmd *cmds, int nr_cmds, bool admin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) for (i = 0; i < nr_cmds; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) nvmet_rdma_free_cmd(ndev, cmds + i, admin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) kfree(cmds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) struct nvmet_rdma_rsp *r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) /* NVMe CQE / RDMA SEND */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) r->req.cqe = kmalloc(sizeof(*r->req.cqe), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (!r->req.cqe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) r->send_sge.addr = ib_dma_map_single(ndev->device, r->req.cqe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) sizeof(*r->req.cqe), DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) if (ib_dma_mapping_error(ndev->device, r->send_sge.addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) goto out_free_rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if (!ib_uses_virt_dma(ndev->device))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) r->req.p2p_client = &ndev->device->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) r->send_sge.length = sizeof(*r->req.cqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) r->send_sge.lkey = ndev->pd->local_dma_lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) r->send_cqe.done = nvmet_rdma_send_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) r->send_wr.wr_cqe = &r->send_cqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) r->send_wr.sg_list = &r->send_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) r->send_wr.num_sge = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) r->send_wr.send_flags = IB_SEND_SIGNALED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) /* Data In / RDMA READ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) r->read_cqe.done = nvmet_rdma_read_data_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) /* Data Out / RDMA WRITE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) r->write_cqe.done = nvmet_rdma_write_data_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) out_free_rsp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) kfree(r->req.cqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) struct nvmet_rdma_rsp *r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) ib_dma_unmap_single(ndev->device, r->send_sge.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) sizeof(*r->req.cqe), DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) kfree(r->req.cqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) struct nvmet_rdma_device *ndev = queue->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) int nr_rsps = queue->recv_queue_size * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) int ret = -EINVAL, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) queue->rsps = kcalloc(nr_rsps, sizeof(struct nvmet_rdma_rsp),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (!queue->rsps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) for (i = 0; i < nr_rsps; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) struct nvmet_rdma_rsp *rsp = &queue->rsps[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) ret = nvmet_rdma_alloc_rsp(ndev, rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) list_add_tail(&rsp->free_list, &queue->free_rsps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) while (--i >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) struct nvmet_rdma_rsp *rsp = &queue->rsps[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) list_del(&rsp->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) nvmet_rdma_free_rsp(ndev, rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) kfree(queue->rsps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct nvmet_rdma_device *ndev = queue->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) int i, nr_rsps = queue->recv_queue_size * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) for (i = 0; i < nr_rsps; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) struct nvmet_rdma_rsp *rsp = &queue->rsps[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) list_del(&rsp->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) nvmet_rdma_free_rsp(ndev, rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) kfree(queue->rsps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) static int nvmet_rdma_post_recv(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) struct nvmet_rdma_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) ib_dma_sync_single_for_device(ndev->device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) cmd->sge[0].addr, cmd->sge[0].length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (cmd->nsrq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) ret = ib_post_srq_recv(cmd->nsrq->srq, &cmd->wr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) ret = ib_post_recv(cmd->queue->qp, &cmd->wr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (unlikely(ret))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) pr_err("post_recv cmd failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) spin_lock(&queue->rsp_wr_wait_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) while (!list_empty(&queue->rsp_wr_wait_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) struct nvmet_rdma_rsp *rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) rsp = list_entry(queue->rsp_wr_wait_list.next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) struct nvmet_rdma_rsp, wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) list_del(&rsp->wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) spin_unlock(&queue->rsp_wr_wait_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) ret = nvmet_rdma_execute_command(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) spin_lock(&queue->rsp_wr_wait_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) list_add(&rsp->wait_list, &queue->rsp_wr_wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) spin_unlock(&queue->rsp_wr_wait_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) static u16 nvmet_rdma_check_pi_status(struct ib_mr *sig_mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct ib_mr_status mr_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) u16 status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) pr_err("ib_check_mr_status failed, ret %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return NVME_SC_INVALID_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) switch (mr_status.sig_err.err_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) case IB_SIG_BAD_GUARD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) status = NVME_SC_GUARD_CHECK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) case IB_SIG_BAD_REFTAG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) status = NVME_SC_REFTAG_CHECK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) case IB_SIG_BAD_APPTAG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) status = NVME_SC_APPTAG_CHECK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) pr_err("PI error found type %d expected 0x%x vs actual 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) mr_status.sig_err.err_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) mr_status.sig_err.expected,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) mr_status.sig_err.actual);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) static void nvmet_rdma_set_sig_domain(struct blk_integrity *bi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) struct nvme_command *cmd, struct ib_sig_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) u16 control, u8 pi_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) domain->sig_type = IB_SIG_TYPE_T10_DIF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) domain->sig.dif.bg_type = IB_T10DIF_CRC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) domain->sig.dif.pi_interval = 1 << bi->interval_exp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) domain->sig.dif.ref_tag = le32_to_cpu(cmd->rw.reftag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) if (control & NVME_RW_PRINFO_PRCHK_REF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) domain->sig.dif.ref_remap = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) domain->sig.dif.app_tag = le16_to_cpu(cmd->rw.apptag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) domain->sig.dif.apptag_check_mask = le16_to_cpu(cmd->rw.appmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) domain->sig.dif.app_escape = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (pi_type == NVME_NS_DPS_PI_TYPE3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) domain->sig.dif.ref_escape = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) static void nvmet_rdma_set_sig_attrs(struct nvmet_req *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) struct ib_sig_attrs *sig_attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) struct nvme_command *cmd = req->cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) u16 control = le16_to_cpu(cmd->rw.control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) u8 pi_type = req->ns->pi_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) struct blk_integrity *bi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) bi = bdev_get_integrity(req->ns->bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) memset(sig_attrs, 0, sizeof(*sig_attrs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (control & NVME_RW_PRINFO_PRACT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) /* for WRITE_INSERT/READ_STRIP no wire domain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) pi_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* Clear the PRACT bit since HCA will generate/verify the PI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) control &= ~NVME_RW_PRINFO_PRACT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) cmd->rw.control = cpu_to_le16(control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /* PI is added by the HW */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) req->transfer_len += req->metadata_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) /* for WRITE_PASS/READ_PASS both wire/memory domains exist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) pi_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) pi_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (control & NVME_RW_PRINFO_PRCHK_REF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) if (control & NVME_RW_PRINFO_PRCHK_GUARD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) sig_attrs->check_mask |= IB_SIG_CHECK_GUARD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (control & NVME_RW_PRINFO_PRCHK_APP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) static int nvmet_rdma_rw_ctx_init(struct nvmet_rdma_rsp *rsp, u64 addr, u32 key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) struct ib_sig_attrs *sig_attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) struct rdma_cm_id *cm_id = rsp->queue->cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) struct nvmet_req *req = &rsp->req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (req->metadata_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) ret = rdma_rw_ctx_signature_init(&rsp->rw, cm_id->qp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) cm_id->port_num, req->sg, req->sg_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) req->metadata_sg, req->metadata_sg_cnt, sig_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) addr, key, nvmet_data_dir(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) ret = rdma_rw_ctx_init(&rsp->rw, cm_id->qp, cm_id->port_num,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) req->sg, req->sg_cnt, 0, addr, key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) nvmet_data_dir(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) struct rdma_cm_id *cm_id = rsp->queue->cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) struct nvmet_req *req = &rsp->req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) if (req->metadata_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) rdma_rw_ctx_destroy_signature(&rsp->rw, cm_id->qp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) cm_id->port_num, req->sg, req->sg_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) req->metadata_sg, req->metadata_sg_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) nvmet_data_dir(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) rdma_rw_ctx_destroy(&rsp->rw, cm_id->qp, cm_id->port_num,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) req->sg, req->sg_cnt, nvmet_data_dir(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) struct nvmet_rdma_queue *queue = rsp->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) if (rsp->n_rdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) nvmet_rdma_rw_ctx_destroy(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) if (rsp->req.sg != rsp->cmd->inline_sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) nvmet_req_free_sgls(&rsp->req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) nvmet_rdma_process_wr_wait_list(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) nvmet_rdma_put_rsp(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) static void nvmet_rdma_error_comp(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (queue->nvme_sq.ctrl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) * we didn't setup the controller yet in case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) * of admin connect error, just disconnect and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * cleanup the queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) nvmet_rdma_queue_disconnect(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) struct nvmet_rdma_rsp *rsp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) container_of(wc->wr_cqe, struct nvmet_rdma_rsp, send_cqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) struct nvmet_rdma_queue *queue = wc->qp->qp_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) nvmet_rdma_release_rsp(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if (unlikely(wc->status != IB_WC_SUCCESS &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) wc->status != IB_WC_WR_FLUSH_ERR)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) pr_err("SEND for CQE 0x%p failed with status %s (%d).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) nvmet_rdma_error_comp(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) static void nvmet_rdma_queue_response(struct nvmet_req *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) struct nvmet_rdma_rsp *rsp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) container_of(req, struct nvmet_rdma_rsp, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) struct rdma_cm_id *cm_id = rsp->queue->cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) struct ib_send_wr *first_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) if (rsp->flags & NVMET_RDMA_REQ_INVALIDATE_RKEY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) rsp->send_wr.opcode = IB_WR_SEND_WITH_INV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) rsp->send_wr.ex.invalidate_rkey = rsp->invalidate_rkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) rsp->send_wr.opcode = IB_WR_SEND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (nvmet_rdma_need_data_out(rsp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) if (rsp->req.metadata_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) cm_id->port_num, &rsp->write_cqe, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) cm_id->port_num, NULL, &rsp->send_wr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) first_wr = &rsp->send_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) nvmet_rdma_post_recv(rsp->queue->dev, rsp->cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) ib_dma_sync_single_for_device(rsp->queue->dev->device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) rsp->send_sge.addr, rsp->send_sge.length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (unlikely(ib_post_send(cm_id->qp, first_wr, NULL))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) pr_err("sending cmd response failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) nvmet_rdma_release_rsp(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) struct nvmet_rdma_rsp *rsp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) container_of(wc->wr_cqe, struct nvmet_rdma_rsp, read_cqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) struct nvmet_rdma_queue *queue = wc->qp->qp_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) u16 status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) WARN_ON(rsp->n_rdma <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) atomic_add(rsp->n_rdma, &queue->sq_wr_avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) rsp->n_rdma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (unlikely(wc->status != IB_WC_SUCCESS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) nvmet_rdma_rw_ctx_destroy(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) nvmet_req_uninit(&rsp->req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) nvmet_rdma_release_rsp(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) if (wc->status != IB_WC_WR_FLUSH_ERR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) pr_info("RDMA READ for CQE 0x%p failed with status %s (%d).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) nvmet_rdma_error_comp(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) if (rsp->req.metadata_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) nvmet_rdma_rw_ctx_destroy(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) if (unlikely(status))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) nvmet_req_complete(&rsp->req, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) rsp->req.execute(&rsp->req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) struct nvmet_rdma_rsp *rsp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) container_of(wc->wr_cqe, struct nvmet_rdma_rsp, write_cqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) struct nvmet_rdma_queue *queue = wc->qp->qp_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) struct rdma_cm_id *cm_id = rsp->queue->cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) u16 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) WARN_ON(rsp->n_rdma <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) atomic_add(rsp->n_rdma, &queue->sq_wr_avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) rsp->n_rdma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) if (unlikely(wc->status != IB_WC_SUCCESS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) nvmet_rdma_rw_ctx_destroy(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) nvmet_req_uninit(&rsp->req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) nvmet_rdma_release_rsp(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) if (wc->status != IB_WC_WR_FLUSH_ERR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) pr_info("RDMA WRITE for CQE failed with status %s (%d).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) ib_wc_status_msg(wc->status), wc->status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) nvmet_rdma_error_comp(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * Upon RDMA completion check the signature status
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * - if succeeded send good NVMe response
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * - if failed send bad NVMe response with appropriate error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (unlikely(status))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) rsp->req.cqe->status = cpu_to_le16(status << 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) nvmet_rdma_rw_ctx_destroy(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) if (unlikely(ib_post_send(cm_id->qp, &rsp->send_wr, NULL))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) pr_err("sending cmd response failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) nvmet_rdma_release_rsp(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp *rsp, u32 len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) u64 off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) int sg_count = num_pages(len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) sg = rsp->cmd->inline_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) for (i = 0; i < sg_count; i++, sg++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) if (i < sg_count - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) sg_unmark_end(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) sg_mark_end(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) sg->offset = off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) sg->length = min_t(int, len, PAGE_SIZE - off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) len -= sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) if (!i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) rsp->req.sg = rsp->cmd->inline_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) rsp->req.sg_cnt = sg_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) static u16 nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) struct nvme_sgl_desc *sgl = &rsp->req.cmd->common.dptr.sgl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) u64 off = le64_to_cpu(sgl->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) u32 len = le32_to_cpu(sgl->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (!nvme_is_write(rsp->req.cmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) rsp->req.error_loc =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) offsetof(struct nvme_common_command, opcode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) if (off + len > rsp->queue->dev->inline_data_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) pr_err("invalid inline data offset!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) return NVME_SC_SGL_INVALID_OFFSET | NVME_SC_DNR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) /* no data command? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (!len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) nvmet_rdma_use_inline_sg(rsp, len, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) rsp->flags |= NVMET_RDMA_REQ_INLINE_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) rsp->req.transfer_len += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) struct nvme_keyed_sgl_desc *sgl, bool invalidate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) u64 addr = le64_to_cpu(sgl->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) u32 key = get_unaligned_le32(sgl->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) struct ib_sig_attrs sig_attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) rsp->req.transfer_len = get_unaligned_le24(sgl->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) /* no data command? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if (!rsp->req.transfer_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) if (rsp->req.metadata_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) nvmet_rdma_set_sig_attrs(&rsp->req, &sig_attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) ret = nvmet_req_alloc_sgls(&rsp->req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) if (unlikely(ret < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) goto error_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) ret = nvmet_rdma_rw_ctx_init(rsp, addr, key, &sig_attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) if (unlikely(ret < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) goto error_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) rsp->n_rdma += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) if (invalidate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) rsp->invalidate_rkey = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) rsp->flags |= NVMET_RDMA_REQ_INVALIDATE_RKEY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) error_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) rsp->req.transfer_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) return NVME_SC_INTERNAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) static u16 nvmet_rdma_map_sgl(struct nvmet_rdma_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) struct nvme_keyed_sgl_desc *sgl = &rsp->req.cmd->common.dptr.ksgl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) switch (sgl->type >> 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) case NVME_SGL_FMT_DATA_DESC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) switch (sgl->type & 0xf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) case NVME_SGL_FMT_OFFSET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) return nvmet_rdma_map_sgl_inline(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) pr_err("invalid SGL subtype: %#x\n", sgl->type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) rsp->req.error_loc =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) offsetof(struct nvme_common_command, dptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) case NVME_KEY_SGL_FMT_DATA_DESC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) switch (sgl->type & 0xf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) case NVME_SGL_FMT_ADDRESS | NVME_SGL_FMT_INVALIDATE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) return nvmet_rdma_map_sgl_keyed(rsp, sgl, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) case NVME_SGL_FMT_ADDRESS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) return nvmet_rdma_map_sgl_keyed(rsp, sgl, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) pr_err("invalid SGL subtype: %#x\n", sgl->type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) rsp->req.error_loc =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) offsetof(struct nvme_common_command, dptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) return NVME_SC_INVALID_FIELD | NVME_SC_DNR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) pr_err("invalid SGL type: %#x\n", sgl->type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) rsp->req.error_loc = offsetof(struct nvme_common_command, dptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) return NVME_SC_SGL_INVALID_TYPE | NVME_SC_DNR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) struct nvmet_rdma_queue *queue = rsp->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) if (unlikely(atomic_sub_return(1 + rsp->n_rdma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) &queue->sq_wr_avail) < 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) pr_debug("IB send queue full (needed %d): queue %u cntlid %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) 1 + rsp->n_rdma, queue->idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) queue->nvme_sq.ctrl->cntlid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) if (nvmet_rdma_need_data_in(rsp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if (rdma_rw_ctx_post(&rsp->rw, queue->qp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) queue->cm_id->port_num, &rsp->read_cqe, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) nvmet_req_complete(&rsp->req, NVME_SC_DATA_XFER_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) rsp->req.execute(&rsp->req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) static void nvmet_rdma_handle_command(struct nvmet_rdma_queue *queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) struct nvmet_rdma_rsp *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) u16 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) ib_dma_sync_single_for_cpu(queue->dev->device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) cmd->cmd->sge[0].addr, cmd->cmd->sge[0].length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) ib_dma_sync_single_for_cpu(queue->dev->device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) cmd->send_sge.addr, cmd->send_sge.length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) if (!nvmet_req_init(&cmd->req, &queue->nvme_cq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) &queue->nvme_sq, &nvmet_rdma_ops))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) status = nvmet_rdma_map_sgl(cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) if (status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) if (unlikely(!nvmet_rdma_execute_command(cmd))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) spin_lock(&queue->rsp_wr_wait_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) list_add_tail(&cmd->wait_list, &queue->rsp_wr_wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) spin_unlock(&queue->rsp_wr_wait_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) out_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) nvmet_req_complete(&cmd->req, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) struct nvmet_rdma_cmd *cmd =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) container_of(wc->wr_cqe, struct nvmet_rdma_cmd, cqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) struct nvmet_rdma_queue *queue = wc->qp->qp_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) struct nvmet_rdma_rsp *rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) if (unlikely(wc->status != IB_WC_SUCCESS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) if (wc->status != IB_WC_WR_FLUSH_ERR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) pr_err("RECV for CQE 0x%p failed with status %s (%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) wc->wr_cqe, ib_wc_status_msg(wc->status),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) wc->status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) nvmet_rdma_error_comp(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) if (unlikely(wc->byte_len < sizeof(struct nvme_command))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) pr_err("Ctrl Fatal Error: capsule size less than 64 bytes\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) nvmet_rdma_error_comp(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) cmd->queue = queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) rsp = nvmet_rdma_get_rsp(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) if (unlikely(!rsp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * we get here only under memory pressure,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) * silently drop and have the host retry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) * as we can't even fail it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) nvmet_rdma_post_recv(queue->dev, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) rsp->queue = queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) rsp->cmd = cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) rsp->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) rsp->req.cmd = cmd->nvme_cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) rsp->req.port = queue->port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) rsp->n_rdma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) if (unlikely(queue->state != NVMET_RDMA_Q_LIVE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) spin_lock_irqsave(&queue->state_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) if (queue->state == NVMET_RDMA_Q_CONNECTING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) list_add_tail(&rsp->wait_list, &queue->rsp_wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) nvmet_rdma_put_rsp(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) spin_unlock_irqrestore(&queue->state_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) nvmet_rdma_handle_command(queue, rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) static void nvmet_rdma_destroy_srq(struct nvmet_rdma_srq *nsrq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) nvmet_rdma_free_cmds(nsrq->ndev, nsrq->cmds, nsrq->ndev->srq_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) ib_destroy_srq(nsrq->srq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) kfree(nsrq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) static void nvmet_rdma_destroy_srqs(struct nvmet_rdma_device *ndev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) if (!ndev->srqs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) for (i = 0; i < ndev->srq_count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) nvmet_rdma_destroy_srq(ndev->srqs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) kfree(ndev->srqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) static struct nvmet_rdma_srq *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) nvmet_rdma_init_srq(struct nvmet_rdma_device *ndev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) struct ib_srq_init_attr srq_attr = { NULL, };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) size_t srq_size = ndev->srq_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) struct nvmet_rdma_srq *nsrq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) struct ib_srq *srq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) int ret, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) nsrq = kzalloc(sizeof(*nsrq), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) if (!nsrq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) srq_attr.attr.max_wr = srq_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) srq_attr.attr.max_sge = 1 + ndev->inline_page_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) srq_attr.attr.srq_limit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) srq_attr.srq_type = IB_SRQT_BASIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) srq = ib_create_srq(ndev->pd, &srq_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) if (IS_ERR(srq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) ret = PTR_ERR(srq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) nsrq->cmds = nvmet_rdma_alloc_cmds(ndev, srq_size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) if (IS_ERR(nsrq->cmds)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) ret = PTR_ERR(nsrq->cmds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) goto out_destroy_srq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) nsrq->srq = srq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) nsrq->ndev = ndev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) for (i = 0; i < srq_size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) nsrq->cmds[i].nsrq = nsrq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) ret = nvmet_rdma_post_recv(ndev, &nsrq->cmds[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) goto out_free_cmds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) return nsrq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) out_free_cmds:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) nvmet_rdma_free_cmds(ndev, nsrq->cmds, srq_size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) out_destroy_srq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) ib_destroy_srq(srq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) kfree(nsrq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) static int nvmet_rdma_init_srqs(struct nvmet_rdma_device *ndev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) if (!ndev->device->attrs.max_srq_wr || !ndev->device->attrs.max_srq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) * If SRQs aren't supported we just go ahead and use normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) * non-shared receive queues.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) pr_info("SRQ requested but not supported.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) ndev->srq_size = min(ndev->device->attrs.max_srq_wr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) nvmet_rdma_srq_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) ndev->srq_count = min(ndev->device->num_comp_vectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) ndev->device->attrs.max_srq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) ndev->srqs = kcalloc(ndev->srq_count, sizeof(*ndev->srqs), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) if (!ndev->srqs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) for (i = 0; i < ndev->srq_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) ndev->srqs[i] = nvmet_rdma_init_srq(ndev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) if (IS_ERR(ndev->srqs[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) ret = PTR_ERR(ndev->srqs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) goto err_srq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) err_srq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) while (--i >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) nvmet_rdma_destroy_srq(ndev->srqs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) kfree(ndev->srqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) static void nvmet_rdma_free_dev(struct kref *ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) struct nvmet_rdma_device *ndev =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) container_of(ref, struct nvmet_rdma_device, ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) mutex_lock(&device_list_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) list_del(&ndev->entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) mutex_unlock(&device_list_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) nvmet_rdma_destroy_srqs(ndev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) ib_dealloc_pd(ndev->pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) kfree(ndev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) static struct nvmet_rdma_device *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) struct nvmet_rdma_port *port = cm_id->context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) struct nvmet_port *nport = port->nport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) struct nvmet_rdma_device *ndev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) int inline_page_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) int inline_sge_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) mutex_lock(&device_list_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) list_for_each_entry(ndev, &device_list, entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) if (ndev->device->node_guid == cm_id->device->node_guid &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) kref_get_unless_zero(&ndev->ref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) if (!ndev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) inline_page_count = num_pages(nport->inline_data_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) inline_sge_count = max(cm_id->device->attrs.max_sge_rd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) cm_id->device->attrs.max_recv_sge) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (inline_page_count > inline_sge_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) pr_warn("inline_data_size %d cannot be supported by device %s. Reducing to %lu.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) nport->inline_data_size, cm_id->device->name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) inline_sge_count * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) nport->inline_data_size = inline_sge_count * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) inline_page_count = inline_sge_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) ndev->inline_data_size = nport->inline_data_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) ndev->inline_page_count = inline_page_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) if (nport->pi_enable && !(cm_id->device->attrs.device_cap_flags &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) IB_DEVICE_INTEGRITY_HANDOVER)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) pr_warn("T10-PI is not supported by device %s. Disabling it\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) cm_id->device->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) nport->pi_enable = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) ndev->device = cm_id->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) kref_init(&ndev->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) ndev->pd = ib_alloc_pd(ndev->device, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) if (IS_ERR(ndev->pd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) goto out_free_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) if (nvmet_rdma_use_srq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) ret = nvmet_rdma_init_srqs(ndev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) goto out_free_pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) list_add(&ndev->entry, &device_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) mutex_unlock(&device_list_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) pr_debug("added %s.\n", ndev->device->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) return ndev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) out_free_pd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) ib_dealloc_pd(ndev->pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) out_free_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) kfree(ndev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) out_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) mutex_unlock(&device_list_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) struct ib_qp_init_attr qp_attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) struct nvmet_rdma_device *ndev = queue->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) int nr_cqe, ret, i, factor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) nr_cqe = queue->recv_queue_size + 2 * queue->send_queue_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) queue->cq = ib_cq_pool_get(ndev->device, nr_cqe + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) queue->comp_vector, IB_POLL_WORKQUEUE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) if (IS_ERR(queue->cq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) ret = PTR_ERR(queue->cq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) pr_err("failed to create CQ cqe= %d ret= %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) nr_cqe + 1, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) memset(&qp_attr, 0, sizeof(qp_attr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) qp_attr.qp_context = queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) qp_attr.event_handler = nvmet_rdma_qp_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) qp_attr.send_cq = queue->cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) qp_attr.recv_cq = queue->cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) qp_attr.qp_type = IB_QPT_RC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) /* +1 for drain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) qp_attr.cap.max_send_wr = queue->send_queue_size + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) factor = rdma_rw_mr_factor(ndev->device, queue->cm_id->port_num,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) 1 << NVMET_RDMA_MAX_MDTS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) qp_attr.cap.max_rdma_ctxs = queue->send_queue_size * factor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) qp_attr.cap.max_send_sge = max(ndev->device->attrs.max_sge_rd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) ndev->device->attrs.max_send_sge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) if (queue->nsrq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) qp_attr.srq = queue->nsrq->srq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) /* +1 for drain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) qp_attr.cap.max_recv_wr = 1 + queue->recv_queue_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) qp_attr.cap.max_recv_sge = 1 + ndev->inline_page_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) if (queue->port->pi_enable && queue->host_qid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) qp_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) ret = rdma_create_qp(queue->cm_id, ndev->pd, &qp_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) pr_err("failed to create_qp ret= %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) goto err_destroy_cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) queue->qp = queue->cm_id->qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) atomic_set(&queue->sq_wr_avail, qp_attr.cap.max_send_wr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) __func__, queue->cq->cqe, qp_attr.cap.max_send_sge,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) qp_attr.cap.max_send_wr, queue->cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) if (!queue->nsrq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) for (i = 0; i < queue->recv_queue_size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) queue->cmds[i].queue = queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) ret = nvmet_rdma_post_recv(ndev, &queue->cmds[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) goto err_destroy_qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) err_destroy_qp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) rdma_destroy_qp(queue->cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) err_destroy_cq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) ib_cq_pool_put(queue->cq, nr_cqe + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) ib_drain_qp(queue->qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) if (queue->cm_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) rdma_destroy_id(queue->cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) ib_destroy_qp(queue->qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) ib_cq_pool_put(queue->cq, queue->recv_queue_size + 2 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) queue->send_queue_size + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) static void nvmet_rdma_free_queue(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) pr_debug("freeing queue %d\n", queue->idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) nvmet_sq_destroy(&queue->nvme_sq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) nvmet_rdma_destroy_queue_ib(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) if (!queue->nsrq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) nvmet_rdma_free_cmds(queue->dev, queue->cmds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) queue->recv_queue_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) !queue->host_qid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) nvmet_rdma_free_rsps(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) ida_simple_remove(&nvmet_rdma_queue_ida, queue->idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) kfree(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) static void nvmet_rdma_release_queue_work(struct work_struct *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) struct nvmet_rdma_queue *queue =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) container_of(w, struct nvmet_rdma_queue, release_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) struct nvmet_rdma_device *dev = queue->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) nvmet_rdma_free_queue(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) kref_put(&dev->ref, nvmet_rdma_free_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param *conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) struct nvme_rdma_cm_req *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) req = (struct nvme_rdma_cm_req *)conn->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) if (!req || conn->private_data_len == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) return NVME_RDMA_CM_INVALID_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) if (le16_to_cpu(req->recfmt) != NVME_RDMA_CM_FMT_1_0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) return NVME_RDMA_CM_INVALID_RECFMT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) queue->host_qid = le16_to_cpu(req->qid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) * req->hsqsize corresponds to our recv queue size plus 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) * req->hrqsize corresponds to our send queue size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) queue->recv_queue_size = le16_to_cpu(req->hsqsize) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) queue->send_queue_size = le16_to_cpu(req->hrqsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) if (!queue->host_qid && queue->recv_queue_size > NVME_AQ_DEPTH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) return NVME_RDMA_CM_INVALID_HSQSIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) /* XXX: Should we enforce some kind of max for IO queues? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) static int nvmet_rdma_cm_reject(struct rdma_cm_id *cm_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) enum nvme_rdma_cm_status status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) struct nvme_rdma_cm_rej rej;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) pr_debug("rejecting connect request: status %d (%s)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) status, nvme_rdma_cm_msg(status));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) rej.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) rej.sts = cpu_to_le16(status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) return rdma_reject(cm_id, (void *)&rej, sizeof(rej),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) IB_CM_REJ_CONSUMER_DEFINED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) static struct nvmet_rdma_queue *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) struct rdma_cm_id *cm_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) struct rdma_cm_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) struct nvmet_rdma_port *port = cm_id->context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) struct nvmet_rdma_queue *queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) queue = kzalloc(sizeof(*queue), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) if (!queue) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) ret = NVME_RDMA_CM_NO_RSC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) goto out_reject;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) ret = nvmet_sq_init(&queue->nvme_sq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) ret = NVME_RDMA_CM_NO_RSC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) goto out_free_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) ret = nvmet_rdma_parse_cm_connect_req(&event->param.conn, queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) goto out_destroy_sq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) * Schedules the actual release because calling rdma_destroy_id from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) * inside a CM callback would trigger a deadlock. (great API design..)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) INIT_WORK(&queue->release_work, nvmet_rdma_release_queue_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) queue->dev = ndev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) queue->cm_id = cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) queue->port = port->nport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) spin_lock_init(&queue->state_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) queue->state = NVMET_RDMA_Q_CONNECTING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) INIT_LIST_HEAD(&queue->rsp_wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) INIT_LIST_HEAD(&queue->rsp_wr_wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) spin_lock_init(&queue->rsp_wr_wait_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) INIT_LIST_HEAD(&queue->free_rsps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) spin_lock_init(&queue->rsps_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) INIT_LIST_HEAD(&queue->queue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) queue->idx = ida_simple_get(&nvmet_rdma_queue_ida, 0, 0, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) if (queue->idx < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) ret = NVME_RDMA_CM_NO_RSC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) goto out_destroy_sq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) * Spread the io queues across completion vectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) * but still keep all admin queues on vector 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) queue->comp_vector = !queue->host_qid ? 0 :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) queue->idx % ndev->device->num_comp_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) ret = nvmet_rdma_alloc_rsps(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) ret = NVME_RDMA_CM_NO_RSC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) goto out_ida_remove;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) if (ndev->srqs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) queue->nsrq = ndev->srqs[queue->comp_vector % ndev->srq_count];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) queue->cmds = nvmet_rdma_alloc_cmds(ndev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) queue->recv_queue_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) !queue->host_qid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) if (IS_ERR(queue->cmds)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) ret = NVME_RDMA_CM_NO_RSC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) goto out_free_responses;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) ret = nvmet_rdma_create_queue_ib(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) pr_err("%s: creating RDMA queue failed (%d).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) __func__, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) ret = NVME_RDMA_CM_NO_RSC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) goto out_free_cmds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) return queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) out_free_cmds:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) if (!queue->nsrq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) nvmet_rdma_free_cmds(queue->dev, queue->cmds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) queue->recv_queue_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) !queue->host_qid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) out_free_responses:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) nvmet_rdma_free_rsps(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) out_ida_remove:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) ida_simple_remove(&nvmet_rdma_queue_ida, queue->idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) out_destroy_sq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) nvmet_sq_destroy(&queue->nvme_sq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) out_free_queue:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) kfree(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) out_reject:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) nvmet_rdma_cm_reject(cm_id, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) static void nvmet_rdma_qp_event(struct ib_event *event, void *priv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) struct nvmet_rdma_queue *queue = priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) switch (event->event) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) case IB_EVENT_COMM_EST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) rdma_notify(queue->cm_id, event->event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) case IB_EVENT_QP_LAST_WQE_REACHED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) pr_debug("received last WQE reached event for queue=0x%p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) pr_err("received IB QP event: %s (%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) ib_event_msg(event->event), event->event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) static int nvmet_rdma_cm_accept(struct rdma_cm_id *cm_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) struct nvmet_rdma_queue *queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) struct rdma_conn_param *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) struct rdma_conn_param param = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) struct nvme_rdma_cm_rep priv = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) param.rnr_retry_count = 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) param.flow_control = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) param.initiator_depth = min_t(u8, p->initiator_depth,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) queue->dev->device->attrs.max_qp_init_rd_atom);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) param.private_data = &priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) param.private_data_len = sizeof(priv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) priv.crqsize = cpu_to_le16(queue->recv_queue_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) ret = rdma_accept(cm_id, ¶m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) pr_err("rdma_accept failed (error code = %d)\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) struct rdma_cm_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) struct nvmet_rdma_device *ndev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) struct nvmet_rdma_queue *queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) ndev = nvmet_rdma_find_get_device(cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) if (!ndev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) nvmet_rdma_cm_reject(cm_id, NVME_RDMA_CM_NO_RSC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) return -ECONNREFUSED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) queue = nvmet_rdma_alloc_queue(ndev, cm_id, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) if (!queue) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) goto put_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) if (queue->host_qid == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) /* Let inflight controller teardown complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) flush_scheduled_work();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) * Don't destroy the cm_id in free path, as we implicitly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) * destroy the cm_id here with non-zero ret code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) queue->cm_id = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) goto free_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) mutex_lock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) list_add_tail(&queue->queue_list, &nvmet_rdma_queue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) mutex_unlock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) free_queue:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) nvmet_rdma_free_queue(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) put_device:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) kref_put(&ndev->ref, nvmet_rdma_free_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) static void nvmet_rdma_queue_established(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) spin_lock_irqsave(&queue->state_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) if (queue->state != NVMET_RDMA_Q_CONNECTING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) pr_warn("trying to establish a connected queue\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) queue->state = NVMET_RDMA_Q_LIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) while (!list_empty(&queue->rsp_wait_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) struct nvmet_rdma_rsp *cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) cmd = list_first_entry(&queue->rsp_wait_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) struct nvmet_rdma_rsp, wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) list_del(&cmd->wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) spin_unlock_irqrestore(&queue->state_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) nvmet_rdma_handle_command(queue, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) spin_lock_irqsave(&queue->state_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) spin_unlock_irqrestore(&queue->state_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) bool disconnect = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) pr_debug("cm_id= %p queue->state= %d\n", queue->cm_id, queue->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) spin_lock_irqsave(&queue->state_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) switch (queue->state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) case NVMET_RDMA_Q_CONNECTING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) while (!list_empty(&queue->rsp_wait_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) struct nvmet_rdma_rsp *rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) rsp = list_first_entry(&queue->rsp_wait_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) struct nvmet_rdma_rsp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) list_del(&rsp->wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) nvmet_rdma_put_rsp(rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) case NVMET_RDMA_Q_LIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) queue->state = NVMET_RDMA_Q_DISCONNECTING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) disconnect = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) case NVMET_RDMA_Q_DISCONNECTING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) spin_unlock_irqrestore(&queue->state_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) if (disconnect) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) rdma_disconnect(queue->cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) schedule_work(&queue->release_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) bool disconnect = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) mutex_lock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) if (!list_empty(&queue->queue_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) list_del_init(&queue->queue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) disconnect = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) mutex_unlock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) if (disconnect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) __nvmet_rdma_queue_disconnect(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) WARN_ON_ONCE(queue->state != NVMET_RDMA_Q_CONNECTING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) mutex_lock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) if (!list_empty(&queue->queue_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) list_del_init(&queue->queue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) mutex_unlock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) pr_err("failed to connect queue %d\n", queue->idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) schedule_work(&queue->release_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) * nvme_rdma_device_removal() - Handle RDMA device removal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) * @cm_id: rdma_cm id, used for nvmet port
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) * @queue: nvmet rdma queue (cm id qp_context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) * DEVICE_REMOVAL event notifies us that the RDMA device is about
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) * to unplug. Note that this event can be generated on a normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) * queue cm_id and/or a device bound listener cm_id (where in this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) * case queue will be null).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) * We registered an ib_client to handle device removal for queues,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) * so we only need to handle the listening port cm_ids. In this case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) * we nullify the priv to prevent double cm_id destruction and destroying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) * the cm_id implicitely by returning a non-zero rc to the callout.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) struct nvmet_rdma_queue *queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) struct nvmet_rdma_port *port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) if (queue) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) * This is a queue cm_id. we have registered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) * an ib_client to handle queues removal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) * so don't interfear and just return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) port = cm_id->context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) * This is a listener cm_id. Make sure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) * future remove_port won't invoke a double
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) * cm_id destroy. use atomic xchg to make sure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) * we don't compete with remove_port.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) if (xchg(&port->cm_id, NULL) != cm_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) * We need to return 1 so that the core will destroy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) * it's own ID. What a great API design..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) struct rdma_cm_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) struct nvmet_rdma_queue *queue = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) if (cm_id->qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) queue = cm_id->qp->qp_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) pr_debug("%s (%d): status %d id %p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) rdma_event_msg(event->event), event->event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) event->status, cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) switch (event->event) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) case RDMA_CM_EVENT_CONNECT_REQUEST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) ret = nvmet_rdma_queue_connect(cm_id, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) case RDMA_CM_EVENT_ESTABLISHED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) nvmet_rdma_queue_established(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) case RDMA_CM_EVENT_ADDR_CHANGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) if (!queue) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) struct nvmet_rdma_port *port = cm_id->context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) schedule_delayed_work(&port->repair_work, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) case RDMA_CM_EVENT_DISCONNECTED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) case RDMA_CM_EVENT_TIMEWAIT_EXIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) nvmet_rdma_queue_disconnect(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) case RDMA_CM_EVENT_DEVICE_REMOVAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) ret = nvmet_rdma_device_removal(cm_id, queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) case RDMA_CM_EVENT_REJECTED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) pr_debug("Connection rejected: %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) rdma_reject_msg(cm_id, event->status));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) case RDMA_CM_EVENT_UNREACHABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) case RDMA_CM_EVENT_CONNECT_ERROR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) nvmet_rdma_queue_connect_fail(cm_id, queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) pr_err("received unrecognized RDMA CM event %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) event->event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) static void nvmet_rdma_delete_ctrl(struct nvmet_ctrl *ctrl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) struct nvmet_rdma_queue *queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) restart:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) mutex_lock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) list_for_each_entry(queue, &nvmet_rdma_queue_list, queue_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) if (queue->nvme_sq.ctrl == ctrl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) list_del_init(&queue->queue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) mutex_unlock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) __nvmet_rdma_queue_disconnect(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) goto restart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) mutex_unlock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) static void nvmet_rdma_destroy_port_queues(struct nvmet_rdma_port *port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) struct nvmet_rdma_queue *queue, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) struct nvmet_port *nport = port->nport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) mutex_lock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) queue_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) if (queue->port != nport)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) list_del_init(&queue->queue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) __nvmet_rdma_queue_disconnect(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) mutex_unlock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) static void nvmet_rdma_disable_port(struct nvmet_rdma_port *port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) struct rdma_cm_id *cm_id = xchg(&port->cm_id, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) if (cm_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) rdma_destroy_id(cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) * Destroy the remaining queues, which are not belong to any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) * controller yet. Do it here after the RDMA-CM was destroyed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) * guarantees that no new queue will be created.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) nvmet_rdma_destroy_port_queues(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) static int nvmet_rdma_enable_port(struct nvmet_rdma_port *port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) struct sockaddr *addr = (struct sockaddr *)&port->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) struct rdma_cm_id *cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) cm_id = rdma_create_id(&init_net, nvmet_rdma_cm_handler, port,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) RDMA_PS_TCP, IB_QPT_RC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) if (IS_ERR(cm_id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) pr_err("CM ID creation failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) return PTR_ERR(cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) * Allow both IPv4 and IPv6 sockets to bind a single port
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) * at the same time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) ret = rdma_set_afonly(cm_id, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) pr_err("rdma_set_afonly failed (%d)\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) goto out_destroy_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) ret = rdma_bind_addr(cm_id, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) pr_err("binding CM ID to %pISpcs failed (%d)\n", addr, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) goto out_destroy_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) ret = rdma_listen(cm_id, 128);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) pr_err("listening to %pISpcs failed (%d)\n", addr, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) goto out_destroy_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) port->cm_id = cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) out_destroy_id:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) rdma_destroy_id(cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) static void nvmet_rdma_repair_port_work(struct work_struct *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) struct nvmet_rdma_port *port = container_of(to_delayed_work(w),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) struct nvmet_rdma_port, repair_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) nvmet_rdma_disable_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) ret = nvmet_rdma_enable_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) schedule_delayed_work(&port->repair_work, 5 * HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) static int nvmet_rdma_add_port(struct nvmet_port *nport)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) struct nvmet_rdma_port *port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) __kernel_sa_family_t af;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) port = kzalloc(sizeof(*port), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) if (!port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) nport->priv = port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) port->nport = nport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) INIT_DELAYED_WORK(&port->repair_work, nvmet_rdma_repair_port_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) switch (nport->disc_addr.adrfam) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) case NVMF_ADDR_FAMILY_IP4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) af = AF_INET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) case NVMF_ADDR_FAMILY_IP6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) af = AF_INET6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) pr_err("address family %d not supported\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) nport->disc_addr.adrfam);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) goto out_free_port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) if (nport->inline_data_size < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) nport->inline_data_size = NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) } else if (nport->inline_data_size > NVMET_RDMA_MAX_INLINE_DATA_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) pr_warn("inline_data_size %u is too large, reducing to %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) nport->inline_data_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) NVMET_RDMA_MAX_INLINE_DATA_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) nport->inline_data_size = NVMET_RDMA_MAX_INLINE_DATA_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) ret = inet_pton_with_scope(&init_net, af, nport->disc_addr.traddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) nport->disc_addr.trsvcid, &port->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) pr_err("malformed ip/port passed: %s:%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) nport->disc_addr.traddr, nport->disc_addr.trsvcid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) goto out_free_port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) ret = nvmet_rdma_enable_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) goto out_free_port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) pr_info("enabling port %d (%pISpcs)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) le16_to_cpu(nport->disc_addr.portid),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) (struct sockaddr *)&port->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) out_free_port:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) kfree(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) static void nvmet_rdma_remove_port(struct nvmet_port *nport)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) struct nvmet_rdma_port *port = nport->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) cancel_delayed_work_sync(&port->repair_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) nvmet_rdma_disable_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) kfree(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) static void nvmet_rdma_disc_port_addr(struct nvmet_req *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) struct nvmet_port *nport, char *traddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) struct nvmet_rdma_port *port = nport->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) struct rdma_cm_id *cm_id = port->cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) if (inet_addr_is_any((struct sockaddr *)&cm_id->route.addr.src_addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) struct nvmet_rdma_rsp *rsp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) container_of(req, struct nvmet_rdma_rsp, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) struct rdma_cm_id *req_cm_id = rsp->queue->cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) struct sockaddr *addr = (void *)&req_cm_id->route.addr.src_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) sprintf(traddr, "%pISc", addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) memcpy(traddr, nport->disc_addr.traddr, NVMF_TRADDR_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) static u8 nvmet_rdma_get_mdts(const struct nvmet_ctrl *ctrl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) if (ctrl->pi_support)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) return NVMET_RDMA_MAX_METADATA_MDTS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) return NVMET_RDMA_MAX_MDTS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) static const struct nvmet_fabrics_ops nvmet_rdma_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) .type = NVMF_TRTYPE_RDMA,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) .msdbd = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) .flags = NVMF_KEYED_SGLS | NVMF_METADATA_SUPPORTED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) .add_port = nvmet_rdma_add_port,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) .remove_port = nvmet_rdma_remove_port,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) .queue_response = nvmet_rdma_queue_response,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) .delete_ctrl = nvmet_rdma_delete_ctrl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) .disc_traddr = nvmet_rdma_disc_port_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) .get_mdts = nvmet_rdma_get_mdts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) struct nvmet_rdma_queue *queue, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) struct nvmet_rdma_device *ndev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) bool found = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) mutex_lock(&device_list_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) list_for_each_entry(ndev, &device_list, entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) if (ndev->device == ib_device) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) found = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) mutex_unlock(&device_list_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) if (!found)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) * IB Device that is used by nvmet controllers is being removed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) * delete all queues using this device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) mutex_lock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) queue_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) if (queue->dev->device != ib_device)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) pr_info("Removing queue %d\n", queue->idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) list_del_init(&queue->queue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) __nvmet_rdma_queue_disconnect(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) mutex_unlock(&nvmet_rdma_queue_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) flush_scheduled_work();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) static struct ib_client nvmet_rdma_ib_client = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) .name = "nvmet_rdma",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) .remove = nvmet_rdma_remove_one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) static int __init nvmet_rdma_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) ret = ib_register_client(&nvmet_rdma_ib_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) ret = nvmet_register_transport(&nvmet_rdma_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) goto err_ib_client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) err_ib_client:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) ib_unregister_client(&nvmet_rdma_ib_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) static void __exit nvmet_rdma_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) nvmet_unregister_transport(&nvmet_rdma_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) ib_unregister_client(&nvmet_rdma_ib_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) WARN_ON_ONCE(!list_empty(&nvmet_rdma_queue_list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) ida_destroy(&nvmet_rdma_queue_ida);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) module_init(nvmet_rdma_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) module_exit(nvmet_rdma_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) MODULE_ALIAS("nvmet-transport-1"); /* 1 == NVMF_TRTYPE_RDMA */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags