^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright(c) 2016 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is provided under a dual BSD/GPLv2 license. When using or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * redistributing this file, you may do so under either license.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * GPL LICENSE SUMMARY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * it under the terms of version 2 of the GNU General Public License as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * BSD LICENSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * modification, are permitted provided that the following conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * - Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * - Redistributions in binary form must reproduce the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * notice, this list of conditions and the following disclaimer in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * the documentation and/or other materials provided with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * - Neither the name of Intel Corporation nor the names of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * contributors may be used to endorse or promote products derived
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <rdma/rdmavt_qp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <rdma/ib_hdrs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * Convert the AETH credit code into the number of credits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static const u16 credit_table[31] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) 0, /* 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) 1, /* 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) 2, /* 2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) 3, /* 3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) 4, /* 4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) 6, /* 5 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) 8, /* 6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) 12, /* 7 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) 16, /* 8 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) 24, /* 9 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) 32, /* A */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) 48, /* B */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) 64, /* C */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) 96, /* D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) 128, /* E */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) 192, /* F */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) 256, /* 10 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) 384, /* 11 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) 512, /* 12 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) 768, /* 13 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) 1024, /* 14 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) 1536, /* 15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) 2048, /* 16 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) 3072, /* 17 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) 4096, /* 18 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) 6144, /* 19 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) 8192, /* 1A */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) 12288, /* 1B */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) 16384, /* 1C */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) 24576, /* 1D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) 32768 /* 1E */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * rvt_compute_aeth - compute the AETH (syndrome + MSN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * @qp: the queue pair to compute the AETH for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * Returns the AETH.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) __be32 rvt_compute_aeth(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) u32 aeth = qp->r_msn & IB_MSN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) if (qp->ibqp.srq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * Shared receive queues don't generate credits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * Set the credit field to the invalid value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) u32 min, max, x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) u32 credits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) u32 head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) u32 tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) credits = READ_ONCE(qp->r_rq.kwq->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (credits == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /* sanity check pointers before trusting them */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (qp->ip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) head = READ_ONCE(qp->r_rq.kwq->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) tail = READ_ONCE(qp->r_rq.kwq->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (head >= qp->r_rq.size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) head = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (tail >= qp->r_rq.size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) tail = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * Compute the number of credits available (RWQEs).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * There is a small chance that the pair of reads are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * not atomic, which is OK, since the fuzziness is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * resolved as further ACKs go out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) credits = rvt_get_rq_count(&qp->r_rq, head, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * Binary search the credit table to find the code to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) min = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) max = 31;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) x = (min + max) / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) if (credit_table[x] == credits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (credit_table[x] > credits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) max = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (min == x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) min = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) aeth |= x << IB_AETH_CREDIT_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) return cpu_to_be32(aeth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) EXPORT_SYMBOL(rvt_compute_aeth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * rvt_get_credit - flush the send work queue of a QP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * @qp: the qp who's send work queue to flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * @aeth: the Acknowledge Extended Transport Header
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * The QP s_lock should be held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) lockdep_assert_held(&qp->s_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * If the credit is invalid, we can send
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * as many packets as we like. Otherwise, we have to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * honor the credit field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (credit == IB_AETH_CREDIT_INVAL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) rdi->driver_f.schedule_send(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) /* Compute new LSN (i.e., MSN + credit) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) qp->s_lsn = credit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) rdi->driver_f.schedule_send(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) EXPORT_SYMBOL(rvt_get_credit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * rvt_restart_sge - rewind the sge state for a wqe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * @ss: the sge state pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * @wqe: the wqe to rewind
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * @len: the data length from the start of the wqe in bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * Returns the remaining data length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) ss->sge = wqe->sg_list[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) ss->sg_list = wqe->sg_list + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) ss->num_sge = wqe->wr.num_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) ss->total_len = wqe->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) rvt_skip_sge(ss, len, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) return wqe->length - len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) EXPORT_SYMBOL(rvt_restart_sge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)