^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright(c) 2015 - 2020 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is provided under a dual BSD/GPLv2 license. When using or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * redistributing this file, you may do so under either license.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * GPL LICENSE SUMMARY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * it under the terms of version 2 of the GNU General Public License as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * BSD LICENSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * modification, are permitted provided that the following conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * - Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * - Redistributions in binary form must reproduce the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * notice, this list of conditions and the following disclaimer in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * the documentation and/or other materials provided with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * - Neither the name of Intel Corporation nor the names of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * contributors may be used to endorse or promote products derived
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <rdma/rdma_vt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <rdma/rdmavt_qp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <rdma/ib_verbs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include "hfi.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include "qp.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include "verbs_txreq.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) unsigned int hfi1_qp_table_size = 256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) module_param_named(qp_table_size, hfi1_qp_table_size, uint, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) MODULE_PARM_DESC(qp_table_size, "QP table size");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static void flush_tx_list(struct rvt_qp *qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static int iowait_sleep(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct sdma_engine *sde,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct iowait_work *wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct sdma_txreq *stx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) unsigned int seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) bool pkts_sent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static void iowait_wakeup(struct iowait *wait, int reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) static void iowait_sdma_drained(struct iowait *wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static void qp_pio_drain(struct rvt_qp *qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) const struct rvt_operation_params hfi1_post_parms[RVT_OPERATION_MAX] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) [IB_WR_RDMA_WRITE] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) .length = sizeof(struct ib_rdma_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) [IB_WR_RDMA_READ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) .length = sizeof(struct ib_rdma_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) .qpt_support = BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) .flags = RVT_OPERATION_ATOMIC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) [IB_WR_ATOMIC_CMP_AND_SWP] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) .length = sizeof(struct ib_atomic_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) .qpt_support = BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) .flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) [IB_WR_ATOMIC_FETCH_AND_ADD] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) .length = sizeof(struct ib_atomic_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) .qpt_support = BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) .flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) [IB_WR_RDMA_WRITE_WITH_IMM] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) .length = sizeof(struct ib_rdma_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) [IB_WR_SEND] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) .length = sizeof(struct ib_send_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) .qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) [IB_WR_SEND_WITH_IMM] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) .length = sizeof(struct ib_send_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) .qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) [IB_WR_REG_MR] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) .length = sizeof(struct ib_reg_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) .flags = RVT_OPERATION_LOCAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) [IB_WR_LOCAL_INV] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) .length = sizeof(struct ib_send_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) .flags = RVT_OPERATION_LOCAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) [IB_WR_SEND_WITH_INV] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) .length = sizeof(struct ib_send_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) .qpt_support = BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) [IB_WR_OPFN] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) .length = sizeof(struct ib_atomic_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) .qpt_support = BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) .flags = RVT_OPERATION_USE_RESERVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) [IB_WR_TID_RDMA_WRITE] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) .length = sizeof(struct ib_rdma_wr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) .qpt_support = BIT(IB_QPT_RC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) .flags = RVT_OPERATION_IGN_RNR_CNT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) static void flush_list_head(struct list_head *l)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) while (!list_empty(l)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) struct sdma_txreq *tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) tx = list_first_entry(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) l,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) struct sdma_txreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) list_del_init(&tx->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) hfi1_put_txreq(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) container_of(tx, struct verbs_txreq, txreq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static void flush_tx_list(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) flush_list_head(&iowait_get_ib_work(&priv->s_iowait)->tx_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) flush_list_head(&iowait_get_tid_work(&priv->s_iowait)->tx_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static void flush_iowait(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) seqlock_t *lock = priv->s_iowait.lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (!lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) write_seqlock_irqsave(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) if (!list_empty(&priv->s_iowait.list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) list_del_init(&priv->s_iowait.list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) priv->s_iowait.lock = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) rvt_put_qp(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) write_sequnlock_irqrestore(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * This function is what we would push to the core layer if we wanted to be a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * "first class citizen". Instead we hide this here and rely on Verbs ULPs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * to blindly pass the MTU enum value from the PathRecord to us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) static inline int verbs_mtu_enum_to_int(struct ib_device *dev, enum ib_mtu mtu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /* Constraining 10KB packets to 8KB packets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (mtu == (enum ib_mtu)OPA_MTU_10240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) mtu = (enum ib_mtu)OPA_MTU_8192;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) return opa_mtu_enum_to_int((enum opa_mtu)mtu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) int attr_mask, struct ib_udata *udata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) struct ib_qp *ibqp = &qp->ibqp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) struct hfi1_ibdev *dev = to_idev(ibqp->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) struct hfi1_devdata *dd = dd_from_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) u8 sc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (attr_mask & IB_QP_AV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) sc = ah_to_sc(ibqp->device, &attr->ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (sc == 0xf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) if (!qp_to_sdma_engine(qp, sc) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) dd->flags & HFI1_HAS_SEND_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) if (!qp_to_send_context(qp, sc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (attr_mask & IB_QP_ALT_PATH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) sc = ah_to_sc(ibqp->device, &attr->alt_ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (sc == 0xf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (!qp_to_sdma_engine(qp, sc) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) dd->flags & HFI1_HAS_SEND_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (!qp_to_send_context(qp, sc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * qp_set_16b - Set the hdr_type based on whether the slid or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * dlid in the connection is extended. Only applicable for RC and UC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * QPs. UD QPs determine this on the fly from the ah in the wqe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static inline void qp_set_16b(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) struct hfi1_pportdata *ppd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) struct hfi1_ibport *ibp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /* Update ah_attr to account for extended LIDs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) hfi1_update_ah_attr(qp->ibqp.device, &qp->remote_ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /* Create 32 bit LIDs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) hfi1_make_opa_lid(&qp->remote_ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) ibp = to_iport(qp->ibqp.device, qp->port_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) ppd = ppd_from_ibp(ibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) priv->hdr_type = hfi1_get_hdr_type(ppd->lid, &qp->remote_ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) int attr_mask, struct ib_udata *udata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct ib_qp *ibqp = &qp->ibqp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (attr_mask & IB_QP_AV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) qp_set_16b(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (attr_mask & IB_QP_PATH_MIG_STATE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) attr->path_mig_state == IB_MIG_MIGRATED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) qp->s_mig_state == IB_MIG_ARMED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) qp->s_flags |= HFI1_S_AHG_CLEAR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) qp_set_16b(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) opfn_qp_init(qp, attr, attr_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * hfi1_setup_wqe - set up the wqe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * @qp - The qp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * @wqe - The built wqe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * @call_send - Determine if the send should be posted or scheduled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * Perform setup of the wqe. This is called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * prior to inserting the wqe into the ring but after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * the wqe has been setup by RDMAVT. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * allows the driver the opportunity to perform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * validation and additional setup of the wqe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * Returns 0 on success, -EINVAL on failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) struct rvt_ah *ah;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) struct hfi1_pportdata *ppd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) struct hfi1_devdata *dd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) switch (qp->ibqp.qp_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) case IB_QPT_RC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) hfi1_setup_tid_rdma_wqe(qp, wqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) case IB_QPT_UC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (wqe->length > 0x80000000U)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (wqe->length > qp->pmtu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) *call_send = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) case IB_QPT_SMI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * SM packets should exclusively use VL15 and their SL is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * ignored (IBTA v1.3, Section 3.5.8.2). Therefore, when ah
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * is created, SL is 0 in most cases and as a result some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * fields (vl and pmtu) in ah may not be set correctly,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * depending on the SL2SC and SC2VL tables at the time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) ppd = ppd_from_ibp(ibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) dd = dd_from_ppd(ppd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (wqe->length > dd->vld[15].mtu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) case IB_QPT_GSI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) case IB_QPT_UD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) ah = rvt_get_swqe_ah(wqe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (wqe->length > (1 << ah->log_pmtu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)] == 0xf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * System latency between send and schedule is large enough that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * forcing call_send to true for piothreshold packets is necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (wqe->length <= piothreshold)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) *call_send = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * _hfi1_schedule_send - schedule progress
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * @qp: the QP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * This schedules qp progress w/o regard to the s_flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) * It is only used in the post send, which doesn't hold
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * the s_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) bool _hfi1_schedule_send(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct hfi1_ibport *ibp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) to_iport(qp->ibqp.device, qp->port_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct hfi1_devdata *dd = ppd->dd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (dd->flags & HFI1_SHUTDOWN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return iowait_schedule(&priv->s_iowait, ppd->hfi1_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) priv->s_sde ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) priv->s_sde->cpu :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) cpumask_first(cpumask_of_node(dd->node)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) static void qp_pio_drain(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (!priv->s_sendcontext)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) while (iowait_pio_pending(&priv->s_iowait)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) write_seqlock_irq(&priv->s_sendcontext->waitlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) write_sequnlock_irq(&priv->s_sendcontext->waitlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) iowait_pio_drain(&priv->s_iowait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) write_seqlock_irq(&priv->s_sendcontext->waitlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) write_sequnlock_irq(&priv->s_sendcontext->waitlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * hfi1_schedule_send - schedule progress
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * @qp: the QP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * This schedules qp progress and caller should hold
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * the s_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * @return true if the first leg is scheduled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * false if the first leg is not scheduled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) bool hfi1_schedule_send(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) lockdep_assert_held(&qp->s_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (hfi1_send_ok(qp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) _hfi1_schedule_send(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) if (qp->s_flags & HFI1_S_ANY_WAIT_IO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) iowait_set_flag(&((struct hfi1_qp_priv *)qp->priv)->s_iowait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) IOWAIT_PENDING_IB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) static void hfi1_qp_schedule(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (iowait_flag_set(&priv->s_iowait, IOWAIT_PENDING_IB)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) ret = hfi1_schedule_send(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) iowait_clear_flag(&priv->s_iowait, IOWAIT_PENDING_IB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) if (iowait_flag_set(&priv->s_iowait, IOWAIT_PENDING_TID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) ret = hfi1_schedule_tid_send(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) iowait_clear_flag(&priv->s_iowait, IOWAIT_PENDING_TID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) spin_lock_irqsave(&qp->s_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (qp->s_flags & flag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) qp->s_flags &= ~flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) trace_hfi1_qpwakeup(qp, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) hfi1_qp_schedule(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) spin_unlock_irqrestore(&qp->s_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) /* Notify hfi1_destroy_qp() if it is waiting. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) rvt_put_qp(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) void hfi1_qp_unbusy(struct rvt_qp *qp, struct iowait_work *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (iowait_set_work_flag(wait) == IOWAIT_IB_SE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) qp->s_flags &= ~RVT_S_BUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * If we are sending a first-leg packet from the second leg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * we need to clear the busy flag from priv->s_flags to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * avoid a race condition when the qp wakes up before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * the call to hfi1_verbs_send() returns to the second
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * leg. In that case, the second leg will terminate without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * being re-scheduled, resulting in failure to send TID RDMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * WRITE DATA and TID RDMA ACK packets.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (priv->s_flags & HFI1_S_TID_BUSY_SET) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) priv->s_flags &= ~(HFI1_S_TID_BUSY_SET |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) RVT_S_BUSY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) iowait_set_flag(&priv->s_iowait, IOWAIT_PENDING_TID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) priv->s_flags &= ~RVT_S_BUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static int iowait_sleep(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) struct sdma_engine *sde,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) struct iowait_work *wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) struct sdma_txreq *stx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) uint seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) bool pkts_sent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) struct verbs_txreq *tx = container_of(stx, struct verbs_txreq, txreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) struct rvt_qp *qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) struct hfi1_qp_priv *priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) qp = tx->qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) spin_lock_irqsave(&qp->s_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) * If we couldn't queue the DMA request, save the info
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * and try again later rather than destroying the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * buffer and undoing the side effects of the copy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /* Make a common routine? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) list_add_tail(&stx->list, &wait->tx_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) write_seqlock(&sde->waitlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (sdma_progress(sde, seq, stx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) goto eagain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (list_empty(&priv->s_iowait.list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) struct hfi1_ibport *ibp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) to_iport(qp->ibqp.device, qp->port_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) ibp->rvp.n_dmawait++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) qp->s_flags |= RVT_S_WAIT_DMA_DESC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) iowait_get_priority(&priv->s_iowait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) iowait_queue(pkts_sent, &priv->s_iowait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) &sde->dmawait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) priv->s_iowait.lock = &sde->waitlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) rvt_get_qp(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) write_sequnlock(&sde->waitlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) hfi1_qp_unbusy(qp, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) spin_unlock_irqrestore(&qp->s_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) spin_unlock_irqrestore(&qp->s_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) hfi1_put_txreq(tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) eagain:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) write_sequnlock(&sde->waitlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) spin_unlock_irqrestore(&qp->s_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) list_del_init(&stx->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) static void iowait_wakeup(struct iowait *wait, int reason)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) struct rvt_qp *qp = iowait_to_qp(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) WARN_ON(reason != SDMA_AVAIL_REASON);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) hfi1_qp_wakeup(qp, RVT_S_WAIT_DMA_DESC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) static void iowait_sdma_drained(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) struct rvt_qp *qp = iowait_to_qp(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * This happens when the send engine notes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * a QP in the error state and cannot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * do the flush work until that QP's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) * sdma work has finished.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) spin_lock_irqsave(&qp->s_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) if (qp->s_flags & RVT_S_WAIT_DMA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) qp->s_flags &= ~RVT_S_WAIT_DMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) hfi1_schedule_send(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) spin_unlock_irqrestore(&qp->s_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) static void hfi1_init_priority(struct iowait *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) struct rvt_qp *qp = iowait_to_qp(w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (qp->s_flags & RVT_S_ACK_PENDING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) w->priority++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) if (priv->s_flags & RVT_S_ACK_PENDING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) w->priority++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * qp_to_sdma_engine - map a qp to a send engine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * @qp: the QP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * @sc5: the 5 bit sc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * Return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) * A send engine for the qp or NULL for SMI type qp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) struct sdma_engine *sde;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (!(dd->flags & HFI1_HAS_SEND_DMA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) switch (qp->ibqp.qp_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) case IB_QPT_SMI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) sde = sdma_select_engine_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, sc5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) return sde;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) * qp_to_send_context - map a qp to a send context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) * @qp: the QP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * @sc5: the 5 bit sc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) * Return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) * A send context for the qp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) switch (qp->ibqp.qp_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) case IB_QPT_SMI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* SMA packets to VL15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) return dd->vld[15].sc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return pio_select_send_context_sc(dd, qp->ibqp.qp_num >> dd->qos_shift,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) sc5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) static const char * const qp_type_str[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) "SMI", "GSI", "RC", "UC", "UD",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) static int qp_idle(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) qp->s_last == qp->s_acked &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) qp->s_acked == qp->s_cur &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) qp->s_cur == qp->s_tail &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) qp->s_tail == qp->s_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * qp_iter_print - print the qp information to seq_file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * @s: the seq_file to emit the qp information on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * @iter: the iterator for the qp hash list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) struct rvt_swqe *wqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) struct rvt_qp *qp = iter->qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) struct sdma_engine *sde;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) struct send_context *send_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) struct rvt_ack_entry *e = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct rvt_srq *srq = qp->ibqp.srq ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) ibsrq_to_rvtsrq(qp->ibqp.srq) : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) sde = qp_to_sdma_engine(qp, priv->s_sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) wqe = rvt_get_swqe_ptr(qp, qp->s_last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) send_context = qp_to_send_context(qp, priv->s_sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) if (qp->s_ack_queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) e = &qp->s_ack_queue[qp->s_tail_ack_queue];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) seq_printf(s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) "N %d %s QP %x R %u %s %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d OS %x %x E %x %x %x RNR %d %s %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) iter->n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) qp_idle(qp) ? "I" : "B",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) qp->ibqp.qp_num,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) atomic_read(&qp->refcount),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) qp_type_str[qp->ibqp.qp_type],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) qp->state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) wqe ? wqe->wr.opcode : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) qp->s_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) iowait_sdma_pending(&priv->s_iowait),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) iowait_pio_pending(&priv->s_iowait),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) !list_empty(&priv->s_iowait.list),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) qp->timeout,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) wqe ? wqe->ssn : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) qp->s_lsn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) qp->s_last_psn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) qp->s_psn, qp->s_next_psn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) qp->s_sending_psn, qp->s_sending_hpsn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) qp->r_psn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) qp->s_last, qp->s_acked, qp->s_cur,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) qp->s_tail, qp->s_head, qp->s_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) qp->s_avail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) /* ack_queue ring pointers, size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) qp->s_tail_ack_queue, qp->r_head_ack_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) rvt_max_atomic(&to_idev(qp->ibqp.device)->rdi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) /* remote QP info */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) qp->remote_qpn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) rdma_ah_get_dlid(&qp->remote_ah_attr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) rdma_ah_get_sl(&qp->remote_ah_attr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) qp->pmtu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) qp->s_retry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) qp->s_retry_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) qp->s_rnr_retry_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) qp->s_rnr_retry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) sde,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) sde ? sde->this_idx : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) send_context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) send_context ? send_context->sw_index : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) ib_cq_head(qp->ibqp.send_cq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) ib_cq_tail(qp->ibqp.send_cq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) qp->pid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) qp->s_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) qp->s_ack_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) /* ack queue information */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) e ? e->opcode : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) e ? e->psn : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) e ? e->lpsn : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) qp->r_min_rnr_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) srq ? "SRQ" : "RQ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) srq ? srq->rq.size : qp->r_rq.size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) struct hfi1_qp_priv *priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) priv = kzalloc_node(sizeof(*priv), GFP_KERNEL, rdi->dparms.node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) if (!priv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) priv->owner = qp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) priv->s_ahg = kzalloc_node(sizeof(*priv->s_ahg), GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) rdi->dparms.node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) if (!priv->s_ahg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) kfree(priv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) iowait_init(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) &priv->s_iowait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) _hfi1_do_send,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) _hfi1_do_tid_send,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) iowait_sleep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) iowait_wakeup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) iowait_sdma_drained,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) hfi1_init_priority);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) /* Init to a value to start the running average correctly */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) priv->s_running_pkt_size = piothreshold / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) void qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) hfi1_qp_priv_tid_free(rdi, qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) kfree(priv->s_ahg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) kfree(priv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) unsigned free_all_qps(struct rvt_dev_info *rdi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) struct hfi1_ibdev *verbs_dev = container_of(rdi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) struct hfi1_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) rdi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) struct hfi1_devdata *dd = container_of(verbs_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) struct hfi1_devdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) verbs_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) int n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) unsigned qp_inuse = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) for (n = 0; n < dd->num_pports; n++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) struct hfi1_ibport *ibp = &dd->pport[n].ibport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (rcu_dereference(ibp->rvp.qp[0]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) qp_inuse++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) if (rcu_dereference(ibp->rvp.qp[1]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) qp_inuse++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return qp_inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) void flush_qp_waiters(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) lockdep_assert_held(&qp->s_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) flush_iowait(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) hfi1_tid_rdma_flush_wait(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) void stop_send_queue(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) iowait_cancel_work(&priv->s_iowait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) if (cancel_work_sync(&priv->tid_rdma.trigger_work))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) rvt_put_qp(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) void quiesce_qp(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) hfi1_del_tid_reap_timer(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) hfi1_del_tid_retry_timer(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) iowait_sdma_drain(&priv->s_iowait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) qp_pio_drain(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) flush_tx_list(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) void notify_qp_reset(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) hfi1_qp_kern_exp_rcv_clear_all(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) qp->r_adefered = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) clear_ahg(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) /* Clear any OPFN state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) if (qp->ibqp.qp_type == IB_QPT_RC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) opfn_conn_error(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) * Switch to alternate path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * The QP s_lock should be held and interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) void hfi1_migrate_qp(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) struct ib_event ev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) qp->s_mig_state = IB_MIG_MIGRATED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) qp->remote_ah_attr = qp->alt_ah_attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) qp->s_pkey_index = qp->s_alt_pkey_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) qp->s_flags |= HFI1_S_AHG_CLEAR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) priv->s_sc = ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) qp_set_16b(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) ev.device = qp->ibqp.device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) ev.element.qp = &qp->ibqp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) ev.event = IB_EVENT_PATH_MIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) int mtu_to_path_mtu(u32 mtu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) return mtu_to_enum(mtu, OPA_MTU_8192);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) u32 mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) u32 mtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) struct hfi1_ibdev *verbs_dev = container_of(rdi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) struct hfi1_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) rdi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) struct hfi1_devdata *dd = container_of(verbs_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) struct hfi1_devdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) verbs_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) struct hfi1_ibport *ibp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) u8 sc, vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) ibp = &dd->pport[qp->port_num - 1].ibport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) sc = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) vl = sc_to_vlt(dd, sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) mtu = verbs_mtu_enum_to_int(qp->ibqp.device, pmtu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) if (vl < PER_VL_SEND_CONTEXTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) mtu = min_t(u32, mtu, dd->vld[vl].mtu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) return mtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) int get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) struct ib_qp_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) int mtu, pidx = qp->port_num - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) struct hfi1_ibdev *verbs_dev = container_of(rdi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) struct hfi1_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) rdi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) struct hfi1_devdata *dd = container_of(verbs_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) struct hfi1_devdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) verbs_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) mtu = verbs_mtu_enum_to_int(qp->ibqp.device, attr->path_mtu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) if (mtu == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) return -1; /* values less than 0 are error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) if (mtu > dd->pport[pidx].ibmtu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) return mtu_to_enum(dd->pport[pidx].ibmtu, IB_MTU_2048);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) return attr->path_mtu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) void notify_error_qp(struct rvt_qp *qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) struct hfi1_qp_priv *priv = qp->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) seqlock_t *lock = priv->s_iowait.lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) if (lock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) write_seqlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (!list_empty(&priv->s_iowait.list) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) !(qp->s_flags & RVT_S_BUSY) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) !(priv->s_flags & RVT_S_BUSY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) qp->s_flags &= ~HFI1_S_ANY_WAIT_IO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) iowait_clear_flag(&priv->s_iowait, IOWAIT_PENDING_IB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) iowait_clear_flag(&priv->s_iowait, IOWAIT_PENDING_TID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) list_del_init(&priv->s_iowait.list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) priv->s_iowait.lock = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) rvt_put_qp(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) write_sequnlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) if (!(qp->s_flags & RVT_S_BUSY) && !(priv->s_flags & RVT_S_BUSY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) qp->s_hdrwords = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) if (qp->s_rdma_mr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) rvt_put_mr(qp->s_rdma_mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) qp->s_rdma_mr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) flush_tx_list(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) * hfi1_qp_iter_cb - callback for iterator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) * @qp - the qp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * @v - the sl in low bits of v
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) * This is called from the iterator callback to work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * on an individual qp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) static void hfi1_qp_iter_cb(struct rvt_qp *qp, u64 v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) int lastwqe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) struct ib_event ev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) struct hfi1_ibport *ibp =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) to_iport(qp->ibqp.device, qp->port_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) u8 sl = (u8)v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) if (qp->port_num != ppd->port ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) (qp->ibqp.qp_type != IB_QPT_UC &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) qp->ibqp.qp_type != IB_QPT_RC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) rdma_ah_get_sl(&qp->remote_ah_attr) != sl ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) !(ib_rvt_state_ops[qp->state] & RVT_POST_SEND_OK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) spin_lock_irq(&qp->r_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) spin_lock(&qp->s_hlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) spin_lock(&qp->s_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) lastwqe = rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) spin_unlock(&qp->s_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) spin_unlock(&qp->s_hlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) spin_unlock_irq(&qp->r_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) if (lastwqe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) ev.device = qp->ibqp.device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) ev.element.qp = &qp->ibqp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * hfi1_error_port_qps - put a port's RC/UC qps into error state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * @ibp: the ibport.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) * @sl: the service level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) * This function places all RC/UC qps with a given service level into error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) * state. It is generally called to force upper lay apps to abandon stale qps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * after an sl->sc mapping change.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) void hfi1_error_port_qps(struct hfi1_ibport *ibp, u8 sl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) struct hfi1_ibdev *dev = &ppd->dd->verbs_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) rvt_qp_iter(&dev->rdi, sl, hfi1_qp_iter_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags