^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/fs/9p/trans_rdma.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * RDMA transport layer based on the trans_fd.c implementation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2008 by Tom Tucker <tom@opengridcomputing.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 2006 by Russ Cox <rsc@swtch.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright (C) 2004-2005 by Latchesar Ionkov <lucho@ionkov.net>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Copyright (C) 2004-2008 by Eric Van Hensbergen <ericvh@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Copyright (C) 1997-2002 by Ron Minnich <rminnich@sarnoff.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/in.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/net.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/ipv6.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/un.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/inet.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/idr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/parser.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/semaphore.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <net/9p/9p.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <net/9p/client.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <net/9p/transport.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <rdma/ib_verbs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <rdma/rdma_cm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define P9_PORT 5640
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define P9_RDMA_SQ_DEPTH 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define P9_RDMA_RQ_DEPTH 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define P9_RDMA_SEND_SGE 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define P9_RDMA_RECV_SGE 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define P9_RDMA_IRD 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define P9_RDMA_ORD 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define P9_RDMA_TIMEOUT 30000 /* 30 seconds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define P9_RDMA_MAXSIZE (1024*1024) /* 1MB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * struct p9_trans_rdma - RDMA transport instance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * @state: tracks the transport state machine for connection setup and tear down
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * @cm_id: The RDMA CM ID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * @pd: Protection Domain pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * @qp: Queue Pair pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * @cq: Completion Queue pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * @timeout: Number of uSecs to wait for connection management events
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * @privport: Whether a privileged port may be used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * @port: The port to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * @sq_depth: The depth of the Send Queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * @sq_sem: Semaphore for the SQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * @rq_depth: The depth of the Receive Queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * @rq_sem: Semaphore for the RQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * @excess_rc : Amount of posted Receive Contexts without a pending request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * See rdma_request()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * @addr: The remote peer's address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * @req_lock: Protects the active request list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * @cm_done: Completion event for connection management tracking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct p9_trans_rdma {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) P9_RDMA_INIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) P9_RDMA_ADDR_RESOLVED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) P9_RDMA_ROUTE_RESOLVED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) P9_RDMA_CONNECTED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) P9_RDMA_FLUSHING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) P9_RDMA_CLOSING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) P9_RDMA_CLOSED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) } state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct rdma_cm_id *cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct ib_pd *pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) struct ib_qp *qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct ib_cq *cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) long timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) bool privport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) u16 port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) int sq_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct semaphore sq_sem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) int rq_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct semaphore rq_sem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) atomic_t excess_rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct sockaddr_in addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) spinlock_t req_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) struct completion cm_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct p9_rdma_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * struct p9_rdma_context - Keeps track of in-process WR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * @busa: Bus address to unmap when the WR completes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * @req: Keeps track of requests (send)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * @rc: Keepts track of replies (receive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) struct p9_rdma_context {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) struct ib_cqe cqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) dma_addr_t busa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) struct p9_req_t *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct p9_fcall rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * struct p9_rdma_opts - Collection of mount options
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * @port: port of connection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * @sq_depth: The requested depth of the SQ. This really doesn't need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * to be any deeper than the number of threads used in the client
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * @rq_depth: The depth of the RQ. Should be greater than or equal to SQ depth
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * @timeout: Time to wait in msecs for CM events
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct p9_rdma_opts {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) short port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) bool privport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) int sq_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) int rq_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) long timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * Option Parsing (code inspired by NFS code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) /* Options that take integer arguments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) Opt_port, Opt_rq_depth, Opt_sq_depth, Opt_timeout,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* Options that take no argument */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) Opt_privport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) Opt_err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static match_table_t tokens = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {Opt_port, "port=%u"},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {Opt_sq_depth, "sq=%u"},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) {Opt_rq_depth, "rq=%u"},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {Opt_timeout, "timeout=%u"},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {Opt_privport, "privport"},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {Opt_err, NULL},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static int p9_rdma_show_options(struct seq_file *m, struct p9_client *clnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct p9_trans_rdma *rdma = clnt->trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if (rdma->port != P9_PORT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) seq_printf(m, ",port=%u", rdma->port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (rdma->sq_depth != P9_RDMA_SQ_DEPTH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) seq_printf(m, ",sq=%u", rdma->sq_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (rdma->rq_depth != P9_RDMA_RQ_DEPTH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) seq_printf(m, ",rq=%u", rdma->rq_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) if (rdma->timeout != P9_RDMA_TIMEOUT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) seq_printf(m, ",timeout=%lu", rdma->timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (rdma->privport)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) seq_puts(m, ",privport");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * parse_opts - parse mount options into rdma options structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * @params: options string passed from mount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * @opts: rdma transport-specific structure to parse options into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * Returns 0 upon success, -ERRNO upon failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static int parse_opts(char *params, struct p9_rdma_opts *opts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) char *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) substring_t args[MAX_OPT_ARGS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) int option;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) char *options, *tmp_options;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) opts->port = P9_PORT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) opts->sq_depth = P9_RDMA_SQ_DEPTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) opts->rq_depth = P9_RDMA_RQ_DEPTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) opts->timeout = P9_RDMA_TIMEOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) opts->privport = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) if (!params)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) tmp_options = kstrdup(params, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) if (!tmp_options) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) p9_debug(P9_DEBUG_ERROR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) "failed to allocate copy of option string\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) options = tmp_options;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) while ((p = strsep(&options, ",")) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) int token;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (!*p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) token = match_token(p, tokens, args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) if ((token != Opt_err) && (token != Opt_privport)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) r = match_int(&args[0], &option);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (r < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) p9_debug(P9_DEBUG_ERROR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) "integer field, but no integer?\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) switch (token) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) case Opt_port:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) opts->port = option;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) case Opt_sq_depth:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) opts->sq_depth = option;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) case Opt_rq_depth:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) opts->rq_depth = option;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) case Opt_timeout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) opts->timeout = option;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) case Opt_privport:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) opts->privport = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /* RQ must be at least as large as the SQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) opts->rq_depth = max(opts->rq_depth, opts->sq_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) kfree(tmp_options);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) p9_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) struct p9_client *c = id->context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) struct p9_trans_rdma *rdma = c->trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) switch (event->event) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) case RDMA_CM_EVENT_ADDR_RESOLVED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) BUG_ON(rdma->state != P9_RDMA_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) rdma->state = P9_RDMA_ADDR_RESOLVED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) case RDMA_CM_EVENT_ROUTE_RESOLVED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) BUG_ON(rdma->state != P9_RDMA_ADDR_RESOLVED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) rdma->state = P9_RDMA_ROUTE_RESOLVED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) case RDMA_CM_EVENT_ESTABLISHED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) BUG_ON(rdma->state != P9_RDMA_ROUTE_RESOLVED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) rdma->state = P9_RDMA_CONNECTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) case RDMA_CM_EVENT_DISCONNECTED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (rdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) rdma->state = P9_RDMA_CLOSED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) c->status = Disconnected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) case RDMA_CM_EVENT_TIMEWAIT_EXIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) case RDMA_CM_EVENT_ADDR_CHANGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) case RDMA_CM_EVENT_ROUTE_ERROR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) case RDMA_CM_EVENT_DEVICE_REMOVAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) case RDMA_CM_EVENT_MULTICAST_JOIN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) case RDMA_CM_EVENT_MULTICAST_ERROR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) case RDMA_CM_EVENT_REJECTED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) case RDMA_CM_EVENT_CONNECT_REQUEST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) case RDMA_CM_EVENT_CONNECT_RESPONSE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) case RDMA_CM_EVENT_CONNECT_ERROR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) case RDMA_CM_EVENT_ADDR_ERROR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) case RDMA_CM_EVENT_UNREACHABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) c->status = Disconnected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) rdma_disconnect(rdma->cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) complete(&rdma->cm_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) recv_done(struct ib_cq *cq, struct ib_wc *wc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) struct p9_client *client = cq->cq_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct p9_trans_rdma *rdma = client->trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct p9_rdma_context *c =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) container_of(wc->wr_cqe, struct p9_rdma_context, cqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) struct p9_req_t *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) int16_t tag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) req = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) ib_dma_unmap_single(rdma->cm_id->device, c->busa, client->msize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (wc->status != IB_WC_SUCCESS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) goto err_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) c->rc.size = wc->byte_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) err = p9_parse_header(&c->rc, NULL, NULL, &tag, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) goto err_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) req = p9_tag_lookup(client, tag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (!req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) goto err_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) /* Check that we have not yet received a reply for this request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (unlikely(req->rc.sdata)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) pr_err("Duplicate reply for request %d", tag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) goto err_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) req->rc.size = c->rc.size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) req->rc.sdata = c->rc.sdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) p9_client_cb(client, req, REQ_STATUS_RCVD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) up(&rdma->rq_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) kfree(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) err_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) p9_debug(P9_DEBUG_ERROR, "req %p err %d status %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) req, err, wc->status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) rdma->state = P9_RDMA_FLUSHING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) client->status = Disconnected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) send_done(struct ib_cq *cq, struct ib_wc *wc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) struct p9_client *client = cq->cq_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct p9_trans_rdma *rdma = client->trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) struct p9_rdma_context *c =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) container_of(wc->wr_cqe, struct p9_rdma_context, cqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) ib_dma_unmap_single(rdma->cm_id->device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) c->busa, c->req->tc.size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) up(&rdma->sq_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) p9_req_put(c->req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) kfree(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) static void qp_event_handler(struct ib_event *event, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) p9_debug(P9_DEBUG_ERROR, "QP event %d context %p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) event->event, context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) static void rdma_destroy_trans(struct p9_trans_rdma *rdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) if (!rdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (rdma->qp && !IS_ERR(rdma->qp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) ib_destroy_qp(rdma->qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (rdma->pd && !IS_ERR(rdma->pd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) ib_dealloc_pd(rdma->pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (rdma->cq && !IS_ERR(rdma->cq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) ib_free_cq(rdma->cq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (rdma->cm_id && !IS_ERR(rdma->cm_id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) rdma_destroy_id(rdma->cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) kfree(rdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) post_recv(struct p9_client *client, struct p9_rdma_context *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) struct p9_trans_rdma *rdma = client->trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) struct ib_recv_wr wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) struct ib_sge sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) c->busa = ib_dma_map_single(rdma->cm_id->device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) c->rc.sdata, client->msize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (ib_dma_mapping_error(rdma->cm_id->device, c->busa))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) c->cqe.done = recv_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) sge.addr = c->busa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) sge.length = client->msize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) sge.lkey = rdma->pd->local_dma_lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) wr.next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) wr.wr_cqe = &c->cqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) wr.sg_list = &sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) wr.num_sge = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) return ib_post_recv(rdma->qp, &wr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) p9_debug(P9_DEBUG_ERROR, "EIO\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) static int rdma_request(struct p9_client *client, struct p9_req_t *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct p9_trans_rdma *rdma = client->trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) struct ib_send_wr wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) struct ib_sge sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) struct p9_rdma_context *c = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) struct p9_rdma_context *rpl_context = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) /* When an error occurs between posting the recv and the send,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * there will be a receive context posted without a pending request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * Since there is no way to "un-post" it, we remember it and skip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * post_recv() for the next request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * So here,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * see if we are this `next request' and need to absorb an excess rc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * If yes, then drop and free our own, and do not recv_post().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (unlikely(atomic_read(&rdma->excess_rc) > 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if ((atomic_sub_return(1, &rdma->excess_rc) >= 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) /* Got one! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) p9_fcall_fini(&req->rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) req->rc.sdata = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) goto dont_need_post_recv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* We raced and lost. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) atomic_inc(&rdma->excess_rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) /* Allocate an fcall for the reply */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) rpl_context = kmalloc(sizeof *rpl_context, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) if (!rpl_context) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) goto recv_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) rpl_context->rc.sdata = req->rc.sdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * Post a receive buffer for this request. We need to ensure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) * there is a reply buffer available for every outstanding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * request. A flushed request can result in no reply for an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * outstanding request, so we must keep a count to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * overflowing the RQ.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) if (down_interruptible(&rdma->rq_sem)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) err = -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) goto recv_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) err = post_recv(client, rpl_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) p9_debug(P9_DEBUG_ERROR, "POST RECV failed: %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) goto recv_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) /* remove posted receive buffer from request structure */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) req->rc.sdata = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) dont_need_post_recv:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) /* Post the request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) c = kmalloc(sizeof *c, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (!c) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) goto send_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) c->req = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) c->busa = ib_dma_map_single(rdma->cm_id->device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) c->req->tc.sdata, c->req->tc.size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) if (ib_dma_mapping_error(rdma->cm_id->device, c->busa)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) goto send_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) c->cqe.done = send_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) sge.addr = c->busa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) sge.length = c->req->tc.size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) sge.lkey = rdma->pd->local_dma_lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) wr.next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) wr.wr_cqe = &c->cqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) wr.opcode = IB_WR_SEND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) wr.send_flags = IB_SEND_SIGNALED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) wr.sg_list = &sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) wr.num_sge = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (down_interruptible(&rdma->sq_sem)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) err = -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) goto send_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) /* Mark request as `sent' *before* we actually send it,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * because doing if after could erase the REQ_STATUS_RCVD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * status in case of a very fast reply.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) req->status = REQ_STATUS_SENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) err = ib_post_send(rdma->qp, &wr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) goto send_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) /* Success */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) /* Handle errors that happened during or while preparing the send: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) send_error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) req->status = REQ_STATUS_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) kfree(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) p9_debug(P9_DEBUG_ERROR, "Error %d in rdma_request()\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /* Ach.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * We did recv_post(), but not send. We have one recv_post in excess.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) atomic_inc(&rdma->excess_rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /* Handle errors that happened during or while preparing post_recv(): */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) recv_error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) kfree(rpl_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) spin_lock_irqsave(&rdma->req_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (err != -EINTR && rdma->state < P9_RDMA_CLOSING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) rdma->state = P9_RDMA_CLOSING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) spin_unlock_irqrestore(&rdma->req_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) rdma_disconnect(rdma->cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) spin_unlock_irqrestore(&rdma->req_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) static void rdma_close(struct p9_client *client)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct p9_trans_rdma *rdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (!client)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) rdma = client->trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) if (!rdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) client->status = Disconnected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) rdma_disconnect(rdma->cm_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) rdma_destroy_trans(rdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * alloc_rdma - Allocate and initialize the rdma transport structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * @opts: Mount options structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) static struct p9_trans_rdma *alloc_rdma(struct p9_rdma_opts *opts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) struct p9_trans_rdma *rdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) rdma = kzalloc(sizeof(struct p9_trans_rdma), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) if (!rdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) rdma->port = opts->port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) rdma->privport = opts->privport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) rdma->sq_depth = opts->sq_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) rdma->rq_depth = opts->rq_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) rdma->timeout = opts->timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) spin_lock_init(&rdma->req_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) init_completion(&rdma->cm_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) sema_init(&rdma->sq_sem, rdma->sq_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) sema_init(&rdma->rq_sem, rdma->rq_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) atomic_set(&rdma->excess_rc, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return rdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) static int rdma_cancel(struct p9_client *client, struct p9_req_t *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /* Nothing to do here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * We will take care of it (if we have to) in rdma_cancelled()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) /* A request has been fully flushed without a reply.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * That means we have posted one buffer in excess.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) static int rdma_cancelled(struct p9_client *client, struct p9_req_t *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) struct p9_trans_rdma *rdma = client->trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) atomic_inc(&rdma->excess_rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) static int p9_rdma_bind_privport(struct p9_trans_rdma *rdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) struct sockaddr_in cl = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) .sin_family = AF_INET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) .sin_addr.s_addr = htonl(INADDR_ANY),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) int port, err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) for (port = P9_DEF_MAX_RESVPORT; port >= P9_DEF_MIN_RESVPORT; port--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) cl.sin_port = htons((ushort)port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) err = rdma_bind_addr(rdma->cm_id, (struct sockaddr *)&cl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if (err != -EADDRINUSE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * rdma_create_trans - Transport method for creating a transport instance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * @client: client instance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * @addr: IP address string
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * @args: Mount options string
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) rdma_create_trans(struct p9_client *client, const char *addr, char *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) struct p9_rdma_opts opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) struct p9_trans_rdma *rdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) struct rdma_conn_param conn_param;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) struct ib_qp_init_attr qp_attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) if (addr == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) /* Parse the transport specific mount options */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) err = parse_opts(args, &opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) /* Create and initialize the RDMA transport structure */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) rdma = alloc_rdma(&opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) if (!rdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) /* Create the RDMA CM ID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) rdma->cm_id = rdma_create_id(&init_net, p9_cm_event_handler, client,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) RDMA_PS_TCP, IB_QPT_RC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (IS_ERR(rdma->cm_id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) /* Associate the client with the transport */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) client->trans = rdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) /* Bind to a privileged port if we need to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (opts.privport) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) err = p9_rdma_bind_privport(rdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) pr_err("%s (%d): problem binding to privport: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) __func__, task_pid_nr(current), -err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) /* Resolve the server's address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) rdma->addr.sin_family = AF_INET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) rdma->addr.sin_addr.s_addr = in_aton(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) rdma->addr.sin_port = htons(opts.port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) err = rdma_resolve_addr(rdma->cm_id, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) (struct sockaddr *)&rdma->addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) rdma->timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) err = wait_for_completion_interruptible(&rdma->cm_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) if (err || (rdma->state != P9_RDMA_ADDR_RESOLVED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) /* Resolve the route to the server */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) err = rdma_resolve_route(rdma->cm_id, rdma->timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) err = wait_for_completion_interruptible(&rdma->cm_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (err || (rdma->state != P9_RDMA_ROUTE_RESOLVED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) /* Create the Completion Queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) rdma->cq = ib_alloc_cq_any(rdma->cm_id->device, client,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) opts.sq_depth + opts.rq_depth + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) IB_POLL_SOFTIRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) if (IS_ERR(rdma->cq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) /* Create the Protection Domain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) rdma->pd = ib_alloc_pd(rdma->cm_id->device, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) if (IS_ERR(rdma->pd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) /* Create the Queue Pair */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) memset(&qp_attr, 0, sizeof qp_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) qp_attr.event_handler = qp_event_handler;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) qp_attr.qp_context = client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) qp_attr.cap.max_send_wr = opts.sq_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) qp_attr.cap.max_recv_wr = opts.rq_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) qp_attr.cap.max_send_sge = P9_RDMA_SEND_SGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) qp_attr.cap.max_recv_sge = P9_RDMA_RECV_SGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) qp_attr.qp_type = IB_QPT_RC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) qp_attr.send_cq = rdma->cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) qp_attr.recv_cq = rdma->cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) err = rdma_create_qp(rdma->cm_id, rdma->pd, &qp_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) rdma->qp = rdma->cm_id->qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) /* Request a connection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) memset(&conn_param, 0, sizeof(conn_param));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) conn_param.private_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) conn_param.private_data_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) conn_param.responder_resources = P9_RDMA_IRD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) conn_param.initiator_depth = P9_RDMA_ORD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) err = rdma_connect(rdma->cm_id, &conn_param);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) err = wait_for_completion_interruptible(&rdma->cm_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (err || (rdma->state != P9_RDMA_CONNECTED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) client->status = Connected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) rdma_destroy_trans(rdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return -ENOTCONN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) static struct p9_trans_module p9_rdma_trans = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) .name = "rdma",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) .maxsize = P9_RDMA_MAXSIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) .def = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) .create = rdma_create_trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) .close = rdma_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) .request = rdma_request,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) .cancel = rdma_cancel,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) .cancelled = rdma_cancelled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) .show_options = p9_rdma_show_options,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * p9_trans_rdma_init - Register the 9P RDMA transport driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) static int __init p9_trans_rdma_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) v9fs_register_trans(&p9_rdma_trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) static void __exit p9_trans_rdma_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) v9fs_unregister_trans(&p9_rdma_trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) module_init(p9_trans_rdma_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) module_exit(p9_trans_rdma_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) MODULE_AUTHOR("Tom Tucker <tom@opengridcomputing.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) MODULE_DESCRIPTION("RDMA Transport for 9P");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) MODULE_LICENSE("Dual BSD/GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags