^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright(c) 2015 - 2018 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is provided under a dual BSD/GPLv2 license. When using or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * redistributing this file, you may do so under either license.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * GPL LICENSE SUMMARY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * it under the terms of version 2 of the GNU General Public License as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * BSD LICENSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * modification, are permitted provided that the following conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * - Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * - Redistributions in binary form must reproduce the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * notice, this list of conditions and the following disclaimer in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * the documentation and/or other materials provided with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * - Neither the name of Intel Corporation nor the names of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * contributors may be used to endorse or promote products derived
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #ifndef HFI1_VERBS_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define HFI1_VERBS_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/seqlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <linux/kref.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <linux/completion.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include <rdma/ib_pack.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include <rdma/ib_user_verbs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #include <rdma/ib_mad.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include <rdma/ib_hdrs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include <rdma/rdma_vt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include <rdma/rdmavt_qp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include <rdma/rdmavt_cq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct hfi1_ctxtdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct hfi1_pportdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct hfi1_devdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct hfi1_packet;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #include "iowait.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #include "tid_rdma.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #include "opfn.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define HFI1_MAX_RDMA_ATOMIC 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * Increment this value if any changes that break userspace ABI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * compatibility are made.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define HFI1_UVERBS_ABI_VERSION 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) /* IB Performance Manager status values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define IB_PMA_SAMPLE_STATUS_DONE 0x00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define IB_PMA_SAMPLE_STATUS_STARTED 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #define IB_PMA_SAMPLE_STATUS_RUNNING 0x02
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /* Mandatory IB performance counter select values. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define IB_PMA_PORT_XMIT_DATA cpu_to_be16(0x0001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define IB_PMA_PORT_RCV_DATA cpu_to_be16(0x0002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define IB_PMA_PORT_XMIT_PKTS cpu_to_be16(0x0003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define IB_PMA_PORT_RCV_PKTS cpu_to_be16(0x0004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define IB_PMA_PORT_XMIT_WAIT cpu_to_be16(0x0005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define HFI1_VENDOR_IPG cpu_to_be16(0xFFA0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #define IB_DEFAULT_GID_PREFIX cpu_to_be64(0xfe80000000000000ULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define OPA_BTH_MIG_REQ BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define RC_OP(x) IB_OPCODE_RC_##x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define UC_OP(x) IB_OPCODE_UC_##x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /* flags passed by hfi1_ib_rcv() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) HFI1_HAS_GRH = (1 << 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define LRH_16B_BYTES (sizeof_field(struct hfi1_16b_header, lrh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define LRH_16B_DWORDS (LRH_16B_BYTES / sizeof(u32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #define LRH_9B_BYTES (sizeof_field(struct ib_header, lrh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define LRH_9B_DWORDS (LRH_9B_BYTES / sizeof(u32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /* 24Bits for qpn, upper 8Bits reserved */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct opa_16b_mgmt {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) __be32 dest_qpn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) __be32 src_qpn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) struct hfi1_16b_header {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) u32 lrh[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct ib_grh grh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct ib_other_headers oth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) } l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) struct ib_other_headers oth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) struct opa_16b_mgmt mgmt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) } u;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) } __packed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct hfi1_opa_header {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) struct ib_header ibh; /* 9B header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct hfi1_16b_header opah; /* 16B header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) u8 hdr_type; /* 9B or 16B */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) } __packed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct hfi1_ahg_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) u32 ahgdesc[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) u16 tx_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) u8 ahgcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) u8 ahgidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct hfi1_sdma_header {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) __le64 pbc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) struct hfi1_opa_header hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) } __packed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * hfi1 specific data structures that will be hidden from rvt after the queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * pair is made common
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) struct hfi1_qp_priv {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct hfi1_ahg_info *s_ahg; /* ahg info for next header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) struct sdma_engine *s_sde; /* current sde */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) struct send_context *s_sendcontext; /* current sendcontext */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) struct hfi1_ctxtdata *rcd; /* QP's receive context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct page **pages; /* for TID page scan */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) u32 tid_enqueue; /* saved when tid waited */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) u8 s_sc; /* SC[0..4] for next packet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct iowait s_iowait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct timer_list s_tid_timer; /* for timing tid wait */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct timer_list s_tid_retry_timer; /* for timing tid ack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct list_head tid_wait; /* for queueing tid space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) struct hfi1_opfn_data opfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct tid_flow_state flow_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) struct tid_rdma_qp_params tid_rdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) struct rvt_qp *owner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) u16 s_running_pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) u8 hdr_type; /* 9B or 16B */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct rvt_sge_state tid_ss; /* SGE state pointer for 2nd leg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) atomic_t n_requests; /* # of TID RDMA requests in the */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /* queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) atomic_t n_tid_requests; /* # of sent TID RDMA requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) unsigned long tid_timer_timeout_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) unsigned long tid_retry_timeout_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) /* variables for the TID RDMA SE state machine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) u8 s_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) u8 s_retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) u8 rnr_nak_state; /* RNR NAK state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) u8 s_nak_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) u32 s_nak_psn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) u32 s_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) u32 s_tid_cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) u32 s_tid_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) u32 s_tid_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) u32 r_tid_head; /* Most recently added TID RDMA request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) u32 r_tid_tail; /* the last completed TID RDMA request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) u32 r_tid_ack; /* the TID RDMA request to be ACK'ed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) u32 r_tid_alloc; /* Request for which we are allocating resources */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) u32 pending_tid_w_segs; /* Num of pending tid write segments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) u32 pending_tid_w_resp; /* Num of pending tid write responses */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) u32 alloc_w_segs; /* Number of segments for which write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* resources have been allocated for this QP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /* For TID RDMA READ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) u32 tid_r_reqs; /* Num of tid reads requested */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) u32 tid_r_comp; /* Num of tid reads completed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) u32 pending_tid_r_segs; /* Num of pending tid read segments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) u16 pkts_ps; /* packets per segment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) u8 timeout_shift; /* account for number of packets per segment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) u32 r_next_psn_kdeth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) u32 r_next_psn_kdeth_save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) u32 s_resync_psn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) u8 sync_pt; /* Set when QP reaches sync point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) u8 resync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) #define HFI1_QP_WQE_INVALID ((u32)-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct hfi1_swqe_priv {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct tid_rdma_request tid_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) struct rvt_sge_state ss; /* Used for TID RDMA READ Request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) struct hfi1_ack_priv {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) struct rvt_sge_state ss; /* used for TID WRITE RESP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) struct tid_rdma_request tid_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * This structure is used to hold commonly lookedup and computed values during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * the send engine progress.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) struct iowait_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct hfi1_pkt_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct hfi1_ibdev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct hfi1_ibport *ibp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct hfi1_pportdata *ppd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) struct verbs_txreq *s_txreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) struct iowait_work *wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) unsigned long timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) unsigned long timeout_int;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) u8 opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) bool in_thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) bool pkts_sent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) #define HFI1_PSN_CREDIT 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct hfi1_opcode_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) u64 n_packets; /* number of packets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) u64 n_bytes; /* total number of bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct hfi1_opcode_stats_perctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) struct hfi1_opcode_stats stats[256];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) static inline void inc_opstats(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) u32 tlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct hfi1_opcode_stats *stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) #ifdef CONFIG_DEBUG_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) stats->n_bytes += tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) stats->n_packets++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct hfi1_ibport {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) struct rvt_qp __rcu *qp[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct rvt_ibport rvp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /* the first 16 entries are sl_to_vl for !OPA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) u8 sl_to_sc[32];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) u8 sc_to_sl[32];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) struct hfi1_ibdev {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct rvt_dev_info rdi; /* Must be first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) /* QP numbers are shared by all IB ports */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) /* protect txwait list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) seqlock_t txwait_lock ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct list_head txwait; /* list for wait verbs_txreq */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) struct list_head memwait; /* list for wait kernel memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) struct kmem_cache *verbs_txreq_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) u64 n_txwait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) u64 n_kmem_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) u64 n_tidwait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /* protect iowait lists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) seqlock_t iowait_lock ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) u64 n_piowait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) u64 n_piodrain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct timer_list mem_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) #ifdef CONFIG_DEBUG_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /* per HFI debugfs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) struct dentry *hfi1_ibdev_dbg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) /* per HFI symlinks to above */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) struct dentry *hfi1_ibdev_link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #ifdef CONFIG_FAULT_INJECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct fault *fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) static inline struct hfi1_ibdev *to_idev(struct ib_device *ibdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) struct rvt_dev_info *rdi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) rdi = container_of(ibdev, struct rvt_dev_info, ibdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) return container_of(rdi, struct hfi1_ibdev, rdi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static inline struct rvt_qp *iowait_to_qp(struct iowait *s_iowait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) struct hfi1_qp_priv *priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) priv = container_of(s_iowait, struct hfi1_qp_priv, s_iowait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) return priv->owner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * This must be called with s_lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) u32 qp1, u32 qp2, u32 lid1, u32 lid2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) void hfi1_sys_guid_chg(struct hfi1_ibport *ibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) void hfi1_node_desc_chg(struct hfi1_ibport *ibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) const struct ib_wc *in_wc, const struct ib_grh *in_grh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) const struct ib_mad *in_mad, struct ib_mad *out_mad,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) size_t *out_mad_size, u16 *out_mad_pkey_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * The PSN_MASK and PSN_SHIFT allow for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * 1) comparing two PSNs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * 2) returning the PSN with any upper bits masked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * 3) returning the difference between to PSNs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * The number of significant bits in the PSN must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * necessarily be at least one bit less than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * the container holding the PSN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) #define PSN_MASK 0x7FFFFFFF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) #define PSN_SHIFT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) #define PSN_MODIFY_MASK 0xFFFFFF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * Compare two PSNs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * Returns an integer <, ==, or > than zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) static inline int cmp_psn(u32 a, u32 b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) return (((int)a) - ((int)b)) << PSN_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * Return masked PSN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) static inline u32 mask_psn(u32 a)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) return a & PSN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * Return delta between two PSNs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) static inline u32 delta_psn(u32 a, u32 b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return (((int)a - (int)b) << PSN_SHIFT) >> PSN_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) static inline struct tid_rdma_request *wqe_to_tid_req(struct rvt_swqe *wqe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) return &((struct hfi1_swqe_priv *)wqe->priv)->tid_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) static inline struct tid_rdma_request *ack_to_tid_req(struct rvt_ack_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) return &((struct hfi1_ack_priv *)e->priv)->tid_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * Look through all the active flows for a TID RDMA request and find
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * the one (if it exists) that contains the specified PSN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) static inline u32 __full_flow_psn(struct flow_state *state, u32 psn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return mask_psn((state->generation << HFI1_KDETH_BTH_SEQ_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) (psn & HFI1_KDETH_BTH_SEQ_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) static inline u32 full_flow_psn(struct tid_rdma_flow *flow, u32 psn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) return __full_flow_psn(&flow->flow_state, psn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) struct verbs_txreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) void hfi1_put_txreq(struct verbs_txreq *tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) void hfi1_cnp_rcv(struct hfi1_packet *packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) void hfi1_uc_rcv(struct hfi1_packet *packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) void hfi1_rc_rcv(struct hfi1_packet *packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) void hfi1_rc_hdrerr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) struct hfi1_ctxtdata *rcd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) struct hfi1_packet *packet,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) struct rvt_qp *qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) void hfi1_ud_rcv(struct hfi1_packet *packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) int hfi1_lookup_pkey_idx(struct hfi1_ibport *ibp, u16 pkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) void hfi1_migrate_qp(struct rvt_qp *qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) int attr_mask, struct ib_udata *udata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) int attr_mask, struct ib_udata *udata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) bool *call_send);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) extern const u32 rc_only_opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) extern const u32 uc_only_opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) const struct ib_global_route *grh, u32 hwords, u32 nwords);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) u32 bth0, u32 bth1, u32 bth2, int middle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) struct hfi1_pkt_state *ps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) bool hfi1_schedule_send_yield(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) bool tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) void _hfi1_do_send(struct work_struct *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) void hfi1_do_send_from_rvt(struct rvt_qp *qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) void hfi1_do_send(struct rvt_qp *qp, bool in_thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) int hfi1_register_ib_device(struct hfi1_devdata *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) void hfi1_unregister_ib_device(struct hfi1_devdata *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) void hfi1_kdeth_eager_rcv(struct hfi1_packet *packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) void hfi1_kdeth_expected_rcv(struct hfi1_packet *packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) void hfi1_ib_rcv(struct hfi1_packet *packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) void hfi1_16B_rcv(struct hfi1_packet *packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) unsigned hfi1_get_npkeys(struct hfi1_devdata *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) u64 pbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) u64 pbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) static inline bool opa_bth_is_migration(struct ib_other_headers *ohdr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) return ohdr->bth[1] & cpu_to_be32(OPA_BTH_MIG_REQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) void hfi1_wait_kmem(struct rvt_qp *qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) static inline void hfi1_trdma_send_complete(struct rvt_qp *qp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) struct rvt_swqe *wqe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) enum ib_wc_status status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) trdma_clean_swqe(qp, wqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) rvt_send_complete(qp, wqe, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) extern const enum ib_wc_opcode ib_hfi1_wc_opcode[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) extern const u8 hdr_len_by_opcode[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) extern const int ib_rvt_state_ops[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) extern __be64 ib_hfi1_sys_image_guid; /* in network order */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) extern unsigned int hfi1_max_cqes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) extern unsigned int hfi1_max_cqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) extern unsigned int hfi1_max_qp_wrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) extern unsigned int hfi1_max_qps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) extern unsigned int hfi1_max_sges;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) extern unsigned int hfi1_max_mcast_grps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) extern unsigned int hfi1_max_mcast_qp_attached;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) extern unsigned int hfi1_max_srqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) extern unsigned int hfi1_max_srq_sges;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) extern unsigned int hfi1_max_srq_wrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) extern unsigned short piothreshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) extern const u32 ib_hfi1_rnr_table[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) #endif /* HFI1_VERBS_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags