^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright(c) 2015 - 2018 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is provided under a dual BSD/GPLv2 license. When using or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * redistributing this file, you may do so under either license.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * GPL LICENSE SUMMARY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * it under the terms of version 2 of the GNU General Public License as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * BSD LICENSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * modification, are permitted provided that the following conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * - Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * - Redistributions in binary form must reproduce the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * notice, this list of conditions and the following disclaimer in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * the documentation and/or other materials provided with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * - Neither the name of Intel Corporation nor the names of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * contributors may be used to endorse or promote products derived
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include "hfi.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include "verbs_txreq.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include "qp.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /* cut down ridiculously long IB macro names */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define OP(x) UC_OP(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * hfi1_make_uc_req - construct a request packet (SEND, RDMA write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * @qp: a pointer to the QP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * Assume s_lock is held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * Return 1 if constructed; otherwise, return 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct ib_other_headers *ohdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct rvt_swqe *wqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) u32 hwords;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) u32 bth0 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) u32 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) u32 pmtu = qp->pmtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) int middle = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) ps->s_txreq = get_txreq(ps->dev, qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (!ps->s_txreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) goto bail_no_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) /* We are in the error state, flush the work request. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (qp->s_last == READ_ONCE(qp->s_head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /* If DMAs are in progress, we can't flush immediately. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) if (iowait_sdma_pending(&priv->s_iowait)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) qp->s_flags |= RVT_S_WAIT_DMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) clear_ahg(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) wqe = rvt_get_swqe_ptr(qp, qp->s_last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) goto done_free_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (priv->hdr_type == HFI1_PKT_TYPE_9B) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) /* header size in 32-bit words LRH+BTH = (8+12)/4. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) hwords = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) /* header size in 32-bit words 16B LRH+BTH = (16+12)/4. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) hwords = 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) ohdr = &ps->s_txreq->phdr.hdr.opah.u.oth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /* Get the next send request. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) qp->s_wqe = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) switch (qp->s_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (!(ib_rvt_state_ops[qp->state] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) RVT_PROCESS_NEXT_SEND_OK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /* Check if send work queue is empty. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (qp->s_cur == READ_ONCE(qp->s_head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) clear_ahg(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * Local operations are processed immediately
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * after all prior requests have completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (wqe->wr.opcode == IB_WR_REG_MR ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) wqe->wr.opcode == IB_WR_LOCAL_INV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) int local_ops = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) if (qp->s_last != qp->s_cur)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (++qp->s_cur == qp->s_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) qp->s_cur = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) err = rvt_invalidate_rkey(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) qp, wqe->wr.ex.invalidate_rkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) local_ops = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) rvt_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) : IB_WC_SUCCESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (local_ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) atomic_dec(&qp->local_ops_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) goto done_free_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * Start a new request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) qp->s_psn = wqe->psn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) qp->s_sge.sge = wqe->sg_list[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) qp->s_sge.sg_list = wqe->sg_list + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) qp->s_sge.num_sge = wqe->wr.num_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) qp->s_sge.total_len = wqe->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) len = wqe->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) qp->s_len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) switch (wqe->wr.opcode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) case IB_WR_SEND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) case IB_WR_SEND_WITH_IMM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (len > pmtu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) qp->s_state = OP(SEND_FIRST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) len = pmtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (wqe->wr.opcode == IB_WR_SEND) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) qp->s_state = OP(SEND_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) qp->s_state =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) OP(SEND_ONLY_WITH_IMMEDIATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /* Immediate data comes after the BTH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) ohdr->u.imm_data = wqe->wr.ex.imm_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) hwords += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) if (wqe->wr.send_flags & IB_SEND_SOLICITED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) bth0 |= IB_BTH_SOLICITED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) qp->s_wqe = wqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (++qp->s_cur >= qp->s_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) qp->s_cur = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) case IB_WR_RDMA_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) case IB_WR_RDMA_WRITE_WITH_IMM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) ohdr->u.rc.reth.vaddr =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) cpu_to_be64(wqe->rdma_wr.remote_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) ohdr->u.rc.reth.rkey =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) cpu_to_be32(wqe->rdma_wr.rkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) ohdr->u.rc.reth.length = cpu_to_be32(len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) hwords += sizeof(struct ib_reth) / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (len > pmtu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) qp->s_state = OP(RDMA_WRITE_FIRST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) len = pmtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) qp->s_state = OP(RDMA_WRITE_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) qp->s_state =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /* Immediate data comes after the RETH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) ohdr->u.rc.imm_data = wqe->wr.ex.imm_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) hwords += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (wqe->wr.send_flags & IB_SEND_SOLICITED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) bth0 |= IB_BTH_SOLICITED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) qp->s_wqe = wqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (++qp->s_cur >= qp->s_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) qp->s_cur = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) case OP(SEND_FIRST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) qp->s_state = OP(SEND_MIDDLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) case OP(SEND_MIDDLE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) len = qp->s_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (len > pmtu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) len = pmtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) middle = HFI1_CAP_IS_KSET(SDMA_AHG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) if (wqe->wr.opcode == IB_WR_SEND) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) qp->s_state = OP(SEND_LAST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) /* Immediate data comes after the BTH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) ohdr->u.imm_data = wqe->wr.ex.imm_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) hwords += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (wqe->wr.send_flags & IB_SEND_SOLICITED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) bth0 |= IB_BTH_SOLICITED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) qp->s_wqe = wqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (++qp->s_cur >= qp->s_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) qp->s_cur = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) case OP(RDMA_WRITE_FIRST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) qp->s_state = OP(RDMA_WRITE_MIDDLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) case OP(RDMA_WRITE_MIDDLE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) len = qp->s_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (len > pmtu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) len = pmtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) middle = HFI1_CAP_IS_KSET(SDMA_AHG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) qp->s_state = OP(RDMA_WRITE_LAST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) qp->s_state =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /* Immediate data comes after the BTH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) ohdr->u.imm_data = wqe->wr.ex.imm_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) hwords += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (wqe->wr.send_flags & IB_SEND_SOLICITED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) bth0 |= IB_BTH_SOLICITED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) qp->s_wqe = wqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (++qp->s_cur >= qp->s_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) qp->s_cur = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) qp->s_len -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) ps->s_txreq->hdr_dwords = hwords;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) ps->s_txreq->sde = priv->s_sde;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) ps->s_txreq->ss = &qp->s_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) ps->s_txreq->s_cur_size = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) hfi1_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) qp->remote_qpn, mask_psn(qp->s_psn++),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) middle, ps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) done_free_tx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) hfi1_put_txreq(ps->s_txreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) ps->s_txreq = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) bail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) hfi1_put_txreq(ps->s_txreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) bail_no_tx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) ps->s_txreq = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) qp->s_flags &= ~RVT_S_BUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * hfi1_uc_rcv - handle an incoming UC packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * @ibp: the port the packet came in on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * @hdr: the header of the packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * @rcv_flags: flags relevant to rcv processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * @data: the packet data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * @tlen: the length of the packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * @qp: the QP for this packet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * This is called from qp_rcv() to process an incoming UC packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * for the given QP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * Called at interrupt level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) void hfi1_uc_rcv(struct hfi1_packet *packet)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) void *data = packet->payload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) u32 tlen = packet->tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) struct rvt_qp *qp = packet->qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) struct ib_other_headers *ohdr = packet->ohdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) u32 opcode = packet->opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) u32 hdrsize = packet->hlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) u32 psn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) u32 pad = packet->pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct ib_wc wc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) u32 pmtu = qp->pmtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct ib_reth *reth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) u8 extra_bytes = pad + packet->extra_byte + (SIZE_OF_CRC << 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (hfi1_ruc_check_hdr(ibp, packet))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) process_ecn(qp, packet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) psn = ib_bth_get_psn(ohdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /* Compare the PSN verses the expected PSN. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (unlikely(cmp_psn(psn, qp->r_psn) != 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * Handle a sequence error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * Silently drop any current message.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) qp->r_psn = psn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) inv:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (qp->r_state == OP(SEND_FIRST) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) qp->r_state == OP(SEND_MIDDLE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) set_bit(RVT_R_REWIND_SGE, &qp->r_aflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) qp->r_sge.num_sge = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) rvt_put_ss(&qp->r_sge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) qp->r_state = OP(SEND_LAST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) switch (opcode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) case OP(SEND_FIRST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) case OP(SEND_ONLY):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) case OP(SEND_ONLY_WITH_IMMEDIATE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) goto send_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) case OP(RDMA_WRITE_FIRST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) case OP(RDMA_WRITE_ONLY):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) goto rdma_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) /* Check for opcode sequence errors. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) switch (qp->r_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) case OP(SEND_FIRST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) case OP(SEND_MIDDLE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) if (opcode == OP(SEND_MIDDLE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) opcode == OP(SEND_LAST) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) opcode == OP(SEND_LAST_WITH_IMMEDIATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) goto inv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) case OP(RDMA_WRITE_FIRST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) case OP(RDMA_WRITE_MIDDLE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (opcode == OP(RDMA_WRITE_MIDDLE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) opcode == OP(RDMA_WRITE_LAST) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) goto inv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if (opcode == OP(SEND_FIRST) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) opcode == OP(SEND_ONLY) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) opcode == OP(SEND_ONLY_WITH_IMMEDIATE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) opcode == OP(RDMA_WRITE_FIRST) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) opcode == OP(RDMA_WRITE_ONLY) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) goto inv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) rvt_comm_est(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) /* OK, process the packet. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) switch (opcode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) case OP(SEND_FIRST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) case OP(SEND_ONLY):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) case OP(SEND_ONLY_WITH_IMMEDIATE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) send_first:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) qp->r_sge = qp->s_rdma_read_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) ret = rvt_get_rwqe(qp, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) goto op_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) * qp->s_rdma_read_sge will be the owner
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) * of the mr references.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) qp->s_rdma_read_sge = qp->r_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) qp->r_rcv_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (opcode == OP(SEND_ONLY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) goto no_immediate_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) goto send_last_imm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) case OP(SEND_MIDDLE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /* Check for invalid length PMTU or posted rwqe len. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * There will be no padding for 9B packet but 16B packets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * will come in with some padding since we always add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * CRC and LT bytes which will need to be flit aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) goto rewind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) qp->r_rcv_len += pmtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (unlikely(qp->r_rcv_len > qp->r_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) goto rewind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) rvt_copy_sge(qp, &qp->r_sge, data, pmtu, false, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) case OP(SEND_LAST_WITH_IMMEDIATE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) send_last_imm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) wc.ex.imm_data = ohdr->u.imm_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) wc.wc_flags = IB_WC_WITH_IMM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) goto send_last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) case OP(SEND_LAST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) no_immediate_data:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) wc.ex.imm_data = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) wc.wc_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) send_last:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) /* Check for invalid length. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /* LAST len should be >= 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) if (unlikely(tlen < (hdrsize + extra_bytes)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) goto rewind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) /* Don't count the CRC. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) tlen -= (hdrsize + extra_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) wc.byte_len = tlen + qp->r_rcv_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (unlikely(wc.byte_len > qp->r_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) goto rewind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) wc.opcode = IB_WC_RECV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) rvt_copy_sge(qp, &qp->r_sge, data, tlen, false, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) rvt_put_ss(&qp->s_rdma_read_sge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) last_imm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) wc.wr_id = qp->r_wr_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) wc.status = IB_WC_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) wc.qp = &qp->ibqp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) wc.src_qp = qp->remote_qpn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * It seems that IB mandates the presence of an SL in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * work completion only for the UD transport (see section
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * 11.4.2 of IBTA Vol. 1).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * However, the way the SL is chosen below is consistent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * with the way that IB/qib works and is trying avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * introducing incompatibilities.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * See also OPA Vol. 1, section 9.7.6, and table 9-17.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) /* zero fields that are N/A */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) wc.vendor_err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) wc.pkey_index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) wc.dlid_path_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) wc.port_num = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) /* Signal completion event if the solicited bit is set. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) case OP(RDMA_WRITE_FIRST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) case OP(RDMA_WRITE_ONLY):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): /* consume RWQE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) rdma_first:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (unlikely(!(qp->qp_access_flags &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) IB_ACCESS_REMOTE_WRITE))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) reth = &ohdr->u.rc.reth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) qp->r_len = be32_to_cpu(reth->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) qp->r_rcv_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) qp->r_sge.sg_list = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (qp->r_len != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) u32 rkey = be32_to_cpu(reth->rkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) u64 vaddr = be64_to_cpu(reth->vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) int ok;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) /* Check rkey */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) vaddr, rkey, IB_ACCESS_REMOTE_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (unlikely(!ok))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) qp->r_sge.num_sge = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) qp->r_sge.num_sge = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) qp->r_sge.sge.mr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) qp->r_sge.sge.vaddr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) qp->r_sge.sge.length = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) qp->r_sge.sge.sge_length = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (opcode == OP(RDMA_WRITE_ONLY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) goto rdma_last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) } else if (opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) wc.ex.imm_data = ohdr->u.rc.imm_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) goto rdma_last_imm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) case OP(RDMA_WRITE_MIDDLE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /* Check for invalid length PMTU or posted rwqe len. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (unlikely(tlen != (hdrsize + pmtu + 4)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) qp->r_rcv_len += pmtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (unlikely(qp->r_rcv_len > qp->r_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) wc.ex.imm_data = ohdr->u.imm_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) rdma_last_imm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) wc.wc_flags = IB_WC_WITH_IMM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) /* Check for invalid length. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) /* LAST len should be >= 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (unlikely(tlen < (hdrsize + pad + 4)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) /* Don't count the CRC. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) tlen -= (hdrsize + extra_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (unlikely(tlen + qp->r_rcv_len != qp->r_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) rvt_put_ss(&qp->s_rdma_read_sge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) ret = rvt_get_rwqe(qp, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) goto op_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) wc.byte_len = qp->r_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) rvt_put_ss(&qp->r_sge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) goto last_imm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) case OP(RDMA_WRITE_LAST):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) rdma_last:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /* Check for invalid length. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) /* LAST len should be >= 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (unlikely(tlen < (hdrsize + pad + 4)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) /* Don't count the CRC. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) tlen -= (hdrsize + extra_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (unlikely(tlen + qp->r_rcv_len != qp->r_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) rvt_put_ss(&qp->r_sge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /* Drop packet for unknown opcodes. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) qp->r_psn++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) qp->r_state = opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) rewind:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) set_bit(RVT_R_REWIND_SGE, &qp->r_aflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) qp->r_sge.num_sge = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) drop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) ibp->rvp.n_pkt_drops++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) op_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags