^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #ifndef _RDS_IB_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #define _RDS_IB_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <rdma/ib_verbs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <rdma/rdma_cm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "rds.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "rdma_transport.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #define RDS_IB_MAX_SGE 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #define RDS_IB_RECV_SGE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #define RDS_IB_DEFAULT_RECV_WR 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define RDS_IB_DEFAULT_SEND_WR 256
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define RDS_IB_DEFAULT_FR_WR 512
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #define RDS_IB_DEFAULT_RETRY_COUNT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define RDS_IB_SUPPORTED_PROTOCOLS 0x00000003 /* minor versions supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define RDS_IB_RECYCLE_BATCH_COUNT 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define RDS_IB_WC_MAX 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) extern struct rw_semaphore rds_ib_devices_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) extern struct list_head rds_ib_devices;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * IB posts RDS_FRAG_SIZE fragments of pages to the receive queues to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * try and minimize the amount of memory tied up both the device and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * socket receive queues.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) struct rds_page_frag {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct list_head f_item;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct list_head f_cache_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct scatterlist f_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct rds_ib_incoming {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct list_head ii_frags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct list_head ii_cache_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct rds_incoming ii_inc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct rds_ib_cache_head {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct list_head *first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) unsigned long count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct rds_ib_refill_cache {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct rds_ib_cache_head __percpu *percpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct list_head *xfer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct list_head *ready;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) /* This is the common structure for the IB private data exchange in setting up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * an RDS connection. The exchange is different for IPv4 and IPv6 connections.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * The reason is that the address size is different and the addresses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * exchanged are in the beginning of the structure. Hence it is not possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * for interoperability if same structure is used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct rds_ib_conn_priv_cmn {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) u8 ricpc_protocol_major;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) u8 ricpc_protocol_minor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) __be16 ricpc_protocol_minor_mask; /* bitmask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) u8 ricpc_dp_toss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) u8 ripc_reserved1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) __be16 ripc_reserved2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) __be64 ricpc_ack_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) __be32 ricpc_credit; /* non-zero enables flow ctl */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct rds_ib_connect_private {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /* Add new fields at the end, and don't permute existing fields. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) __be32 dp_saddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) __be32 dp_daddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct rds_ib_conn_priv_cmn dp_cmn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct rds6_ib_connect_private {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /* Add new fields at the end, and don't permute existing fields. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct in6_addr dp_saddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) struct in6_addr dp_daddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct rds_ib_conn_priv_cmn dp_cmn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define dp_protocol_major dp_cmn.ricpc_protocol_major
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define dp_protocol_minor dp_cmn.ricpc_protocol_minor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define dp_protocol_minor_mask dp_cmn.ricpc_protocol_minor_mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define dp_ack_seq dp_cmn.ricpc_ack_seq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define dp_credit dp_cmn.ricpc_credit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) union rds_ib_conn_priv {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct rds_ib_connect_private ricp_v4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct rds6_ib_connect_private ricp_v6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct rds_ib_send_work {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) void *s_op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct ib_send_wr s_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct ib_rdma_wr s_rdma_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) struct ib_atomic_wr s_atomic_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct ib_sge s_sge[RDS_IB_MAX_SGE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) unsigned long s_queued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) struct rds_ib_recv_work {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) struct rds_ib_incoming *r_ibinc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) struct rds_page_frag *r_frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct ib_recv_wr r_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct ib_sge r_sge[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) struct rds_ib_work_ring {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) u32 w_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) u32 w_alloc_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) u32 w_alloc_ctr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) u32 w_free_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) atomic_t w_free_ctr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /* Rings are posted with all the allocations they'll need to queue the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * incoming message to the receiving socket so this can't fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * All fragments start with a header, so we can make sure we're not receiving
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * garbage, and we can tell a small 8 byte fragment from an ACK frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct rds_ib_ack_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) u64 ack_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) u64 ack_recv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) unsigned int ack_required:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) unsigned int ack_next_valid:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) unsigned int ack_recv_valid:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct rds_ib_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct rds_ib_connection {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct list_head ib_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct rds_ib_device *rds_ibdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct rds_connection *conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /* alphabet soup, IBTA style */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) struct rdma_cm_id *i_cm_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) struct ib_pd *i_pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) struct ib_cq *i_send_cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct ib_cq *i_recv_cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) struct ib_wc i_send_wc[RDS_IB_WC_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) struct ib_wc i_recv_wc[RDS_IB_WC_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) /* To control the number of wrs from fastreg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) atomic_t i_fastreg_wrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) atomic_t i_fastreg_inuse_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) /* interrupt handling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct tasklet_struct i_send_tasklet;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct tasklet_struct i_recv_tasklet;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /* tx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct rds_ib_work_ring i_send_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct rm_data_op *i_data_op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct rds_header **i_send_hdrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) dma_addr_t *i_send_hdrs_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct rds_ib_send_work *i_sends;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) atomic_t i_signaled_sends;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* rx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct mutex i_recv_mutex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct rds_ib_work_ring i_recv_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) struct rds_ib_incoming *i_ibinc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) u32 i_recv_data_rem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) struct rds_header **i_recv_hdrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) dma_addr_t *i_recv_hdrs_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) struct rds_ib_recv_work *i_recvs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) u64 i_ack_recv; /* last ACK received */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) struct rds_ib_refill_cache i_cache_incs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) struct rds_ib_refill_cache i_cache_frags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) atomic_t i_cache_allocs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /* sending acks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) unsigned long i_ack_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) #ifdef KERNEL_HAS_ATOMIC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) atomic64_t i_ack_next; /* next ACK to send */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) spinlock_t i_ack_lock; /* protect i_ack_next */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) u64 i_ack_next; /* next ACK to send */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) struct rds_header *i_ack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) struct ib_send_wr i_ack_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) struct ib_sge i_ack_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) dma_addr_t i_ack_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) unsigned long i_ack_queued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) /* Flow control related information
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * Our algorithm uses a pair variables that we need to access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * atomically - one for the send credits, and one posted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * recv credits we need to transfer to remote.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * Rather than protect them using a slow spinlock, we put both into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * a single atomic_t and update it using cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) atomic_t i_credits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /* Protocol version specific information */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) unsigned int i_flowctl:1; /* enable/disable flow ctl */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) /* Batched completions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) unsigned int i_unsignaled_wrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /* Endpoint role in connection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) bool i_active_side;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) atomic_t i_cq_quiesce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /* Send/Recv vectors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) int i_scq_vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) int i_rcq_vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) u8 i_sl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) /* This assumes that atomic_t is at least 32 bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) #define IB_GET_SEND_CREDITS(v) ((v) & 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) #define IB_GET_POST_CREDITS(v) ((v) >> 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #define IB_SET_SEND_CREDITS(v) ((v) & 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) #define IB_SET_POST_CREDITS(v) ((v) << 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct rds_ib_ipaddr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) __be32 ipaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) RDS_IB_MR_8K_POOL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) RDS_IB_MR_1M_POOL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) struct rds_ib_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) struct list_head ipaddr_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) struct list_head conn_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) struct ib_device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) struct ib_pd *pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) u8 odp_capable:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) unsigned int max_mrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct rds_ib_mr_pool *mr_1m_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) struct rds_ib_mr_pool *mr_8k_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) unsigned int max_8k_mrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) unsigned int max_1m_mrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) int max_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) unsigned int max_wrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) unsigned int max_initiator_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) unsigned int max_responder_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) spinlock_t spinlock; /* protect the above */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) refcount_t refcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) struct work_struct free_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) int *vector_load;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) #define rdsibdev_to_node(rdsibdev) ibdev_to_node(rdsibdev->dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) /* bits for i_ack_flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) #define IB_ACK_IN_FLIGHT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) #define IB_ACK_REQUESTED 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /* Magic WR_ID for ACKs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) #define RDS_IB_ACK_WR_ID (~(u64) 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct rds_ib_statistics {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) uint64_t s_ib_connect_raced;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) uint64_t s_ib_listen_closed_stale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) uint64_t s_ib_evt_handler_call;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) uint64_t s_ib_tasklet_call;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) uint64_t s_ib_tx_cq_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) uint64_t s_ib_tx_ring_full;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) uint64_t s_ib_tx_throttle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) uint64_t s_ib_tx_sg_mapping_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) uint64_t s_ib_tx_stalled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) uint64_t s_ib_tx_credit_updates;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) uint64_t s_ib_rx_cq_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) uint64_t s_ib_rx_ring_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) uint64_t s_ib_rx_refill_from_cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) uint64_t s_ib_rx_refill_from_thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) uint64_t s_ib_rx_alloc_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) uint64_t s_ib_rx_total_frags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) uint64_t s_ib_rx_total_incs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) uint64_t s_ib_rx_credit_updates;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) uint64_t s_ib_ack_sent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) uint64_t s_ib_ack_send_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) uint64_t s_ib_ack_send_delayed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) uint64_t s_ib_ack_send_piggybacked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) uint64_t s_ib_ack_received;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) uint64_t s_ib_rdma_mr_8k_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) uint64_t s_ib_rdma_mr_8k_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) uint64_t s_ib_rdma_mr_8k_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) uint64_t s_ib_rdma_mr_8k_pool_flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) uint64_t s_ib_rdma_mr_8k_pool_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) uint64_t s_ib_rdma_mr_8k_pool_depleted;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) uint64_t s_ib_rdma_mr_1m_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) uint64_t s_ib_rdma_mr_1m_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) uint64_t s_ib_rdma_mr_1m_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) uint64_t s_ib_rdma_mr_1m_pool_flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) uint64_t s_ib_rdma_mr_1m_pool_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) uint64_t s_ib_rdma_mr_1m_pool_depleted;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) uint64_t s_ib_rdma_mr_8k_reused;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) uint64_t s_ib_rdma_mr_1m_reused;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) uint64_t s_ib_atomic_cswp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) uint64_t s_ib_atomic_fadd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) uint64_t s_ib_recv_added_to_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) uint64_t s_ib_recv_removed_from_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) extern struct workqueue_struct *rds_ib_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * Fake ib_dma_sync_sg_for_{cpu,device} as long as ib_verbs.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * doesn't define it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) static inline void rds_ib_dma_sync_sg_for_cpu(struct ib_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) struct scatterlist *sglist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) unsigned int sg_dma_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) int direction)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) for_each_sg(sglist, sg, sg_dma_len, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) ib_dma_sync_single_for_cpu(dev, sg_dma_address(sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) sg_dma_len(sg), direction);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) #define ib_dma_sync_sg_for_cpu rds_ib_dma_sync_sg_for_cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) static inline void rds_ib_dma_sync_sg_for_device(struct ib_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) struct scatterlist *sglist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) unsigned int sg_dma_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) int direction)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) for_each_sg(sglist, sg, sg_dma_len, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) ib_dma_sync_single_for_device(dev, sg_dma_address(sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) sg_dma_len(sg), direction);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) #define ib_dma_sync_sg_for_device rds_ib_dma_sync_sg_for_device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) /* ib.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) extern struct rds_transport rds_ib_transport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) struct rds_ib_device *rds_ib_get_client_data(struct ib_device *device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) void rds_ib_dev_put(struct rds_ib_device *rds_ibdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) extern struct ib_client rds_ib_client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) extern unsigned int rds_ib_retry_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) extern spinlock_t ib_nodev_conns_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) extern struct list_head ib_nodev_conns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /* ib_cm.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) int rds_ib_conn_alloc(struct rds_connection *conn, gfp_t gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) void rds_ib_conn_free(void *arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) int rds_ib_conn_path_connect(struct rds_conn_path *cp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) void rds_ib_conn_path_shutdown(struct rds_conn_path *cp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) void rds_ib_state_change(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) int rds_ib_listen_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) void rds_ib_listen_stop(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) __printf(2, 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) void __rds_ib_conn_error(struct rds_connection *conn, const char *, ...);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) struct rdma_cm_event *event, bool isv6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id, bool isv6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) void rds_ib_cm_connect_complete(struct rds_connection *conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct rdma_cm_event *event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) #define rds_ib_conn_error(conn, fmt...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) __rds_ib_conn_error(conn, KERN_WARNING "RDS/IB: " fmt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) /* ib_rdma.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) int rds_ib_update_ipaddr(struct rds_ib_device *rds_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) struct in6_addr *ipaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) void rds_ib_add_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) void rds_ib_remove_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) void rds_ib_destroy_nodev_conns(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) void rds_ib_mr_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) /* ib_recv.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) int rds_ib_recv_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) void rds_ib_recv_exit(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) int rds_ib_recv_path(struct rds_conn_path *conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic, gfp_t gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) void rds_ib_recv_free_caches(struct rds_ib_connection *ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) void rds_ib_inc_free(struct rds_incoming *inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) struct rds_ib_ack_state *state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) void rds_ib_recv_tasklet_fn(unsigned long data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) void rds_ib_recv_init_ring(struct rds_ib_connection *ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) void rds_ib_recv_clear_ring(struct rds_ib_connection *ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) void rds_ib_recv_init_ack(struct rds_ib_connection *ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) void rds_ib_attempt_ack(struct rds_ib_connection *ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) void rds_ib_ack_send_complete(struct rds_ib_connection *ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) /* ib_ring.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) void rds_ib_ring_init(struct rds_ib_work_ring *ring, u32 nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) void rds_ib_ring_resize(struct rds_ib_work_ring *ring, u32 nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) u32 rds_ib_ring_alloc(struct rds_ib_work_ring *ring, u32 val, u32 *pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) void rds_ib_ring_free(struct rds_ib_work_ring *ring, u32 val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) void rds_ib_ring_unalloc(struct rds_ib_work_ring *ring, u32 val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) int rds_ib_ring_empty(struct rds_ib_work_ring *ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) int rds_ib_ring_low(struct rds_ib_work_ring *ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) u32 rds_ib_ring_oldest(struct rds_ib_work_ring *ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) u32 rds_ib_ring_completed(struct rds_ib_work_ring *ring, u32 wr_id, u32 oldest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) extern wait_queue_head_t rds_ib_ring_empty_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) /* ib_send.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) void rds_ib_xmit_path_complete(struct rds_conn_path *cp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) unsigned int hdr_off, unsigned int sg, unsigned int off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) void rds_ib_send_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) void rds_ib_send_init_ring(struct rds_ib_connection *ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) void rds_ib_send_clear_ring(struct rds_ib_connection *ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) int rds_ib_xmit_rdma(struct rds_connection *conn, struct rm_rdma_op *op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) void rds_ib_send_add_credits(struct rds_connection *conn, unsigned int credits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) void rds_ib_advertise_credits(struct rds_connection *conn, unsigned int posted);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) int rds_ib_send_grab_credits(struct rds_ib_connection *ic, u32 wanted,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) u32 *adv_credits, int need_posted, int max_posted);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) int rds_ib_xmit_atomic(struct rds_connection *conn, struct rm_atomic_op *op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) /* ib_stats.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_ib_statistics, rds_ib_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) #define rds_ib_stats_inc(member) rds_stats_inc_which(rds_ib_stats, member)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) #define rds_ib_stats_add(member, count) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) rds_stats_add_which(rds_ib_stats, member, count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) unsigned int rds_ib_stats_info_copy(struct rds_info_iterator *iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) unsigned int avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) /* ib_sysctl.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) int rds_ib_sysctl_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) void rds_ib_sysctl_exit(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) extern unsigned long rds_ib_sysctl_max_send_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) extern unsigned long rds_ib_sysctl_max_recv_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) extern unsigned long rds_ib_sysctl_max_unsig_wrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) extern unsigned long rds_ib_sysctl_max_unsig_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) extern unsigned long rds_ib_sysctl_max_recv_allocation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) extern unsigned int rds_ib_sysctl_flow_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate