^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright(c) 2016 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is provided under a dual BSD/GPLv2 license. When using or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * redistributing this file, you may do so under either license.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * GPL LICENSE SUMMARY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * it under the terms of version 2 of the GNU General Public License as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * BSD LICENSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * modification, are permitted provided that the following conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * - Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * - Redistributions in binary form must reproduce the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * notice, this list of conditions and the following disclaimer in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * the documentation and/or other materials provided with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * - Neither the name of Intel Corporation nor the names of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * contributors may be used to endorse or promote products derived
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <rdma/ib_umem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <rdma/rdma_vt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include "vt.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include "mr.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * rvt_driver_mr_init - Init MR resources per driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * @rdi: rvt dev struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * Do any intilization needed when a driver registers with rdmavt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * Return: 0 on success or errno on failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) int rvt_driver_mr_init(struct rvt_dev_info *rdi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) unsigned int lkey_table_size = rdi->dparms.lkey_table_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) unsigned lk_tab_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * The top hfi1_lkey_table_size bits are used to index the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * table. The lower 8 bits can be owned by the user (copied from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * the LKEY). The remaining bits act as a generation number or tag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (!lkey_table_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) spin_lock_init(&rdi->lkey_table.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /* ensure generation is at least 4 bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) if (lkey_table_size > RVT_MAX_LKEY_TABLE_BITS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) rvt_pr_warn(rdi, "lkey bits %u too large, reduced to %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) lkey_table_size, RVT_MAX_LKEY_TABLE_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) rdi->dparms.lkey_table_size = RVT_MAX_LKEY_TABLE_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) lkey_table_size = rdi->dparms.lkey_table_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) rdi->lkey_table.max = 1 << lkey_table_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) rdi->lkey_table.shift = 32 - lkey_table_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) lk_tab_size = rdi->lkey_table.max * sizeof(*rdi->lkey_table.table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) rdi->lkey_table.table = (struct rvt_mregion __rcu **)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) vmalloc_node(lk_tab_size, rdi->dparms.node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) if (!rdi->lkey_table.table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) RCU_INIT_POINTER(rdi->dma_mr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) for (i = 0; i < rdi->lkey_table.max; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) RCU_INIT_POINTER(rdi->lkey_table.table[i], NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) rdi->dparms.props.max_mr = rdi->lkey_table.max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) *rvt_mr_exit: clean up MR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) *@rdi: rvt dev structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * called when drivers have unregistered or perhaps failed to register with us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) void rvt_mr_exit(struct rvt_dev_info *rdi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (rdi->dma_mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) rvt_pr_err(rdi, "DMA MR not null!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) vfree(rdi->lkey_table.table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static void rvt_deinit_mregion(struct rvt_mregion *mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) int i = mr->mapsz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) mr->mapsz = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) while (i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) kfree(mr->map[--i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) percpu_ref_exit(&mr->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) static void __rvt_mregion_complete(struct percpu_ref *ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) struct rvt_mregion *mr = container_of(ref, struct rvt_mregion,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) complete(&mr->comp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) int count, unsigned int percpu_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) int m, i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct rvt_dev_info *dev = ib_to_rvt(pd->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) mr->mapsz = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) m = (count + RVT_SEGSZ - 1) / RVT_SEGSZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) for (; i < m; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) mr->map[i] = kzalloc_node(sizeof(*mr->map[0]), GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) dev->dparms.node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (!mr->map[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) mr->mapsz++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) init_completion(&mr->comp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) /* count returning the ptr to user */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (percpu_ref_init(&mr->refcount, &__rvt_mregion_complete,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) percpu_flags, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) atomic_set(&mr->lkey_invalid, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) mr->pd = pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) mr->max_segs = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) bail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) rvt_deinit_mregion(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * rvt_alloc_lkey - allocate an lkey
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * @mr: memory region that this lkey protects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * @dma_region: 0->normal key, 1->restricted DMA key
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * Returns 0 if successful, otherwise returns -errno.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * Increments mr reference count as required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * Sets the lkey field mr for non-dma regions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) u32 r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) u32 n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) struct rvt_dev_info *dev = ib_to_rvt(mr->pd->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct rvt_lkey_table *rkt = &dev->lkey_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) rvt_get_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) spin_lock_irqsave(&rkt->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) /* special case for dma_mr lkey == 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) if (dma_region) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) struct rvt_mregion *tmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) tmr = rcu_access_pointer(dev->dma_mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) if (!tmr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) mr->lkey_published = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /* Insure published written first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) rcu_assign_pointer(dev->dma_mr, mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) rvt_get_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) goto success;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /* Find the next available LKEY */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) r = rkt->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) n = r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (!rcu_access_pointer(rkt->table[r]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) r = (r + 1) & (rkt->max - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (r == n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) rkt->next = (r + 1) & (rkt->max - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * Make sure lkey is never zero which is reserved to indicate an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * unrestricted LKEY.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) rkt->gen++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * bits are capped to ensure enough bits for generation number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) mr->lkey = (r << (32 - dev->dparms.lkey_table_size)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) ((((1 << (24 - dev->dparms.lkey_table_size)) - 1) & rkt->gen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) << 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (mr->lkey == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) mr->lkey |= 1 << 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) rkt->gen++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) mr->lkey_published = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) /* Insure published written first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) rcu_assign_pointer(rkt->table[r], mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) success:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) spin_unlock_irqrestore(&rkt->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) bail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) rvt_put_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) spin_unlock_irqrestore(&rkt->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * rvt_free_lkey - free an lkey
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * @mr: mr to free from tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) static void rvt_free_lkey(struct rvt_mregion *mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) u32 lkey = mr->lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) u32 r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct rvt_dev_info *dev = ib_to_rvt(mr->pd->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) struct rvt_lkey_table *rkt = &dev->lkey_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) int freed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) spin_lock_irqsave(&rkt->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (!lkey) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (mr->lkey_published) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) mr->lkey_published = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) /* insure published is written before pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) rcu_assign_pointer(dev->dma_mr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) rvt_put_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (!mr->lkey_published)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) r = lkey >> (32 - dev->dparms.lkey_table_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) mr->lkey_published = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /* insure published is written before pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) rcu_assign_pointer(rkt->table[r], NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) freed++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) spin_unlock_irqrestore(&rkt->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (freed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) percpu_ref_kill(&mr->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) struct rvt_mr *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) int rval = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) int m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /* Allocate struct plus pointers to first level page tables. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) m = (count + RVT_SEGSZ - 1) / RVT_SEGSZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) mr = kzalloc(struct_size(mr, mr.map, m), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (!mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) rval = rvt_init_mregion(&mr->mr, pd, count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (rval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * ib_reg_phys_mr() will initialize mr->ibmr except for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * lkey and rkey.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) rval = rvt_alloc_lkey(&mr->mr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) if (rval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) goto bail_mregion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) mr->ibmr.lkey = mr->mr.lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) mr->ibmr.rkey = mr->mr.lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) return mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) bail_mregion:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) rvt_deinit_mregion(&mr->mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) bail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) kfree(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) mr = ERR_PTR(rval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) static void __rvt_free_mr(struct rvt_mr *mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) rvt_free_lkey(&mr->mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) rvt_deinit_mregion(&mr->mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) kfree(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * rvt_get_dma_mr - get a DMA memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * @pd: protection domain for this memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * @acc: access flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * Return: the memory region on success, otherwise returns an errno.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) struct ib_mr *rvt_get_dma_mr(struct ib_pd *pd, int acc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) struct rvt_mr *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) struct ib_mr *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) int rval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) if (ibpd_to_rvtpd(pd)->user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return ERR_PTR(-EPERM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) mr = kzalloc(sizeof(*mr), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (!mr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) ret = ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) rval = rvt_init_mregion(&mr->mr, pd, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (rval) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) ret = ERR_PTR(rval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) rval = rvt_alloc_lkey(&mr->mr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (rval) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) ret = ERR_PTR(rval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) goto bail_mregion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) mr->mr.access_flags = acc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) ret = &mr->ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) bail_mregion:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) rvt_deinit_mregion(&mr->mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) bail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) kfree(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * rvt_reg_user_mr - register a userspace memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * @pd: protection domain for this memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * @start: starting userspace address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * @length: length of region to register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * @mr_access_flags: access flags for this memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * @udata: unused by the driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * Return: the memory region on success, otherwise returns an errno.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) u64 virt_addr, int mr_access_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) struct ib_udata *udata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct rvt_mr *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) struct ib_umem *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct sg_page_iter sg_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) int n, m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) struct ib_mr *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (length == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) umem = ib_umem_get(pd->device, start, length, mr_access_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (IS_ERR(umem))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) return (void *)umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) n = ib_umem_num_pages(umem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) mr = __rvt_alloc_mr(n, pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (IS_ERR(mr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) ret = (struct ib_mr *)mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) goto bail_umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) mr->mr.user_base = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) mr->mr.iova = virt_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) mr->mr.length = length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) mr->mr.offset = ib_umem_offset(umem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) mr->mr.access_flags = mr_access_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) mr->umem = umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) mr->mr.page_shift = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) m = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) for_each_sg_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) void *vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) vaddr = page_address(sg_page_iter_page(&sg_iter));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (!vaddr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) ret = ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) goto bail_inval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) mr->mr.map[m]->segs[n].vaddr = vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) mr->mr.map[m]->segs[n].length = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) trace_rvt_mr_user_seg(&mr->mr, m, n, vaddr, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) if (++n == RVT_SEGSZ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) m++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) return &mr->ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) bail_inval:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) __rvt_free_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) bail_umem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) ib_umem_release(umem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * rvt_dereg_clean_qp_cb - callback from iterator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * @qp - the qp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) * @v - the mregion (as u64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * This routine fields the callback for all QPs and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * for QPs in the same PD as the MR will call the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * rvt_qp_mr_clean() to potentially cleanup references.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) static void rvt_dereg_clean_qp_cb(struct rvt_qp *qp, u64 v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) struct rvt_mregion *mr = (struct rvt_mregion *)v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) /* skip PDs that are not ours */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (mr->pd != qp->ibqp.pd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) rvt_qp_mr_clean(qp, mr->lkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * rvt_dereg_clean_qps - find QPs for reference cleanup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * @mr - the MR that is being deregistered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * This routine iterates RC QPs looking for references
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * to the lkey noted in mr.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) static void rvt_dereg_clean_qps(struct rvt_mregion *mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) struct rvt_dev_info *rdi = ib_to_rvt(mr->pd->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) rvt_qp_iter(rdi, (u64)mr, rvt_dereg_clean_qp_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * rvt_check_refs - check references
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * @mr - the megion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * @t - the caller identification
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * This routine checks MRs holding a reference during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * when being de-registered.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * If the count is non-zero, the code calls a clean routine then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * waits for the timeout for the count to zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) static int rvt_check_refs(struct rvt_mregion *mr, const char *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) unsigned long timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) struct rvt_dev_info *rdi = ib_to_rvt(mr->pd->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (mr->lkey) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) /* avoid dma mr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) rvt_dereg_clean_qps(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) /* @mr was indexed on rcu protected @lkey_table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) timeout = wait_for_completion_timeout(&mr->comp, 5 * HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (!timeout) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) rvt_pr_err(rdi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) "%s timeout mr %p pd %p lkey %x refcount %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) t, mr, mr->pd, mr->lkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) atomic_long_read(&mr->refcount.data->count));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) rvt_get_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * rvt_mr_has_lkey - is MR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) * @mr - the mregion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) * @lkey - the lkey
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) bool rvt_mr_has_lkey(struct rvt_mregion *mr, u32 lkey)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) return mr && lkey == mr->lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * rvt_ss_has_lkey - is mr in sge tests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * @ss - the sge state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * @lkey
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) * This code tests for an MR in the indicated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) * sge state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) bool rvt_ss_has_lkey(struct rvt_sge_state *ss, u32 lkey)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) bool rval = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (!ss->num_sge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return rval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) /* first one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) rval = rvt_mr_has_lkey(ss->sge.mr, lkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) /* any others */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) for (i = 0; !rval && i < ss->num_sge - 1; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) rval = rvt_mr_has_lkey(ss->sg_list[i].mr, lkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return rval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) * rvt_dereg_mr - unregister and free a memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * @ibmr: the memory region to free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * Note that this is called to free MRs created by rvt_get_dma_mr()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * or rvt_reg_user_mr().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * Returns 0 on success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) int rvt_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) struct rvt_mr *mr = to_imr(ibmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) rvt_free_lkey(&mr->mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) rvt_put_mr(&mr->mr); /* will set completion if last */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) ret = rvt_check_refs(&mr->mr, __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) rvt_deinit_mregion(&mr->mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) ib_umem_release(mr->umem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) kfree(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * rvt_alloc_mr - Allocate a memory region usable with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * @pd: protection domain for this memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) * @mr_type: mem region type
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * @max_num_sg: Max number of segments allowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * Return: the memory region on success, otherwise return an errno.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) u32 max_num_sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) struct rvt_mr *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) if (mr_type != IB_MR_TYPE_MEM_REG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) mr = __rvt_alloc_mr(max_num_sg, pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (IS_ERR(mr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) return (struct ib_mr *)mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return &mr->ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * rvt_set_page - page assignment function called by ib_sg_to_pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * @ibmr: memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * @addr: dma address of mapped page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) * Return: 0 on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) static int rvt_set_page(struct ib_mr *ibmr, u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) struct rvt_mr *mr = to_imr(ibmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) u32 ps = 1 << mr->mr.page_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) u32 mapped_segs = mr->mr.length >> mr->mr.page_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) int m, n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) if (unlikely(mapped_segs == mr->mr.max_segs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) m = mapped_segs / RVT_SEGSZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) n = mapped_segs % RVT_SEGSZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) mr->mr.map[m]->segs[n].vaddr = (void *)addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) mr->mr.map[m]->segs[n].length = ps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) mr->mr.length += ps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) trace_rvt_mr_page_seg(&mr->mr, m, n, (void *)addr, ps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * rvt_map_mr_sg - map sg list and set it the memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * @ibmr: memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * @sg: dma mapped scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * @sg_nents: number of entries in sg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * @sg_offset: offset in bytes into sg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * Overwrite rvt_mr length with mr length calculated by ib_sg_to_pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * Return: number of sg elements mapped to the memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) int rvt_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) int sg_nents, unsigned int *sg_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) struct rvt_mr *mr = to_imr(ibmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) mr->mr.length = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) mr->mr.page_shift = PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rvt_set_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) mr->mr.user_base = ibmr->iova;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) mr->mr.iova = ibmr->iova;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) mr->mr.offset = ibmr->iova - (u64)mr->mr.map[0]->segs[0].vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) mr->mr.length = (size_t)ibmr->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) trace_rvt_map_mr_sg(ibmr, sg_nents, sg_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) * rvt_fast_reg_mr - fast register physical MR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * @qp: the queue pair where the work request comes from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) * @ibmr: the memory region to be registered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) * @key: updated key for this memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * @access: access flags for this memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * Returns 0 on success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) int rvt_fast_reg_mr(struct rvt_qp *qp, struct ib_mr *ibmr, u32 key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) int access)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) struct rvt_mr *mr = to_imr(ibmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) if (qp->ibqp.pd != mr->mr.pd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) return -EACCES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /* not applicable to dma MR or user MR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) if (!mr->mr.lkey || mr->umem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) if ((key & 0xFFFFFF00) != (mr->mr.lkey & 0xFFFFFF00))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) ibmr->lkey = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) ibmr->rkey = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) mr->mr.lkey = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) mr->mr.access_flags = access;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) mr->mr.iova = ibmr->iova;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) atomic_set(&mr->mr.lkey_invalid, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) EXPORT_SYMBOL(rvt_fast_reg_mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) * rvt_invalidate_rkey - invalidate an MR rkey
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) * @qp: queue pair associated with the invalidate op
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) * @rkey: rkey to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) * Returns 0 on success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) struct rvt_lkey_table *rkt = &dev->lkey_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) struct rvt_mregion *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) if (rkey == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) mr = rcu_dereference(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) rkt->table[(rkey >> (32 - dev->dparms.lkey_table_size))]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (unlikely(!mr || mr->lkey != rkey || qp->ibqp.pd != mr->pd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) atomic_set(&mr->lkey_invalid, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) bail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) EXPORT_SYMBOL(rvt_invalidate_rkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) * rvt_sge_adjacent - is isge compressible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * @last_sge: last outgoing SGE written
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) * @sge: SGE to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) * If adjacent will update last_sge to add length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * Return: true if isge is adjacent to last sge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) static inline bool rvt_sge_adjacent(struct rvt_sge *last_sge,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) struct ib_sge *sge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (last_sge && sge->lkey == last_sge->mr->lkey &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) ((uint64_t)(last_sge->vaddr + last_sge->length) == sge->addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (sge->lkey) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) if (unlikely((sge->addr - last_sge->mr->user_base +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) sge->length > last_sge->mr->length)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) return false; /* overrun, caller will catch */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) last_sge->length += sge->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) last_sge->sge_length += sge->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) trace_rvt_sge_adjacent(last_sge, sge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * rvt_lkey_ok - check IB SGE for validity and initialize
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * @rkt: table containing lkey to check SGE against
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * @pd: protection domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * @isge: outgoing internal SGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * @last_sge: last outgoing SGE written
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * @sge: SGE to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * @acc: access flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) * Check the IB SGE for validity and initialize our internal version
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) * of it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * Increments the reference count when a new sge is stored.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * Return: 0 if compressed, 1 if added , otherwise returns -errno.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) struct rvt_sge *isge, struct rvt_sge *last_sge,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) struct ib_sge *sge, int acc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) struct rvt_mregion *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) unsigned n, m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) size_t off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) * We use LKEY == zero for kernel virtual addresses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * (see rvt_get_dma_mr()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (sge->lkey == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) struct rvt_dev_info *dev = ib_to_rvt(pd->ibpd.device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) if (pd->user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) if (rvt_sge_adjacent(last_sge, sge))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) mr = rcu_dereference(dev->dma_mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) if (!mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) rvt_get_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) isge->mr = mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) isge->vaddr = (void *)sge->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) isge->length = sge->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) isge->sge_length = sge->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) isge->m = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) isge->n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) goto ok;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (rvt_sge_adjacent(last_sge, sge))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) mr = rcu_dereference(rkt->table[sge->lkey >> rkt->shift]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) if (!mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) rvt_get_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) if (!READ_ONCE(mr->lkey_published))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) goto bail_unref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) if (unlikely(atomic_read(&mr->lkey_invalid) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) mr->lkey != sge->lkey || mr->pd != &pd->ibpd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) goto bail_unref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) off = sge->addr - mr->user_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) if (unlikely(sge->addr < mr->user_base ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) off + sge->length > mr->length ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) (mr->access_flags & acc) != acc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) goto bail_unref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) off += mr->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) if (mr->page_shift) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * page sizes are uniform power of 2 so no loop is necessary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * entries_spanned_by_off is the number of times the loop below
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * would have executed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) size_t entries_spanned_by_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) entries_spanned_by_off = off >> mr->page_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) off -= (entries_spanned_by_off << mr->page_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) m = entries_spanned_by_off / RVT_SEGSZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) n = entries_spanned_by_off % RVT_SEGSZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) m = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) while (off >= mr->map[m]->segs[n].length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) off -= mr->map[m]->segs[n].length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) n++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) if (n >= RVT_SEGSZ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) m++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) isge->mr = mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) isge->vaddr = mr->map[m]->segs[n].vaddr + off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) isge->length = mr->map[m]->segs[n].length - off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) isge->sge_length = sge->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) isge->m = m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) isge->n = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) ok:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) trace_rvt_sge_new(isge, sge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) bail_unref:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) rvt_put_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) bail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) EXPORT_SYMBOL(rvt_lkey_ok);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) * rvt_rkey_ok - check the IB virtual address, length, and RKEY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) * @qp: qp for validation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) * @sge: SGE state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) * @len: length of data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) * @vaddr: virtual address to place data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * @rkey: rkey to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) * @acc: access flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) * Return: 1 if successful, otherwise 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * increments the reference count upon success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) u32 len, u64 vaddr, u32 rkey, int acc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) struct rvt_dev_info *dev = ib_to_rvt(qp->ibqp.device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) struct rvt_lkey_table *rkt = &dev->lkey_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) struct rvt_mregion *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) unsigned n, m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) size_t off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * We use RKEY == zero for kernel virtual addresses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * (see rvt_get_dma_mr()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) if (rkey == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) struct rvt_pd *pd = ibpd_to_rvtpd(qp->ibqp.pd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) struct rvt_dev_info *rdi = ib_to_rvt(pd->ibpd.device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) if (pd->user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) mr = rcu_dereference(rdi->dma_mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) if (!mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) rvt_get_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) sge->mr = mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) sge->vaddr = (void *)vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) sge->length = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) sge->sge_length = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) sge->m = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) sge->n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) goto ok;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) mr = rcu_dereference(rkt->table[rkey >> rkt->shift]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) if (!mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) rvt_get_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) /* insure mr read is before test */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) if (!READ_ONCE(mr->lkey_published))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) goto bail_unref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) if (unlikely(atomic_read(&mr->lkey_invalid) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) mr->lkey != rkey || qp->ibqp.pd != mr->pd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) goto bail_unref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) off = vaddr - mr->iova;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) if (unlikely(vaddr < mr->iova || off + len > mr->length ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) (mr->access_flags & acc) == 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) goto bail_unref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) off += mr->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (mr->page_shift) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) * page sizes are uniform power of 2 so no loop is necessary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) * entries_spanned_by_off is the number of times the loop below
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) * would have executed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) size_t entries_spanned_by_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) entries_spanned_by_off = off >> mr->page_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) off -= (entries_spanned_by_off << mr->page_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) m = entries_spanned_by_off / RVT_SEGSZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) n = entries_spanned_by_off % RVT_SEGSZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) m = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) while (off >= mr->map[m]->segs[n].length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) off -= mr->map[m]->segs[n].length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) n++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) if (n >= RVT_SEGSZ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) m++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) sge->mr = mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) sge->vaddr = mr->map[m]->segs[n].vaddr + off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) sge->length = mr->map[m]->segs[n].length - off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) sge->sge_length = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) sge->m = m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) sge->n = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) ok:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) bail_unref:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) rvt_put_mr(mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) bail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) EXPORT_SYMBOL(rvt_rkey_ok);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags