^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This software is available to you under a choice of one of two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * licenses. You may choose to be licensed under the terms of the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * General Public License (GPL) Version 2, available from the file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * COPYING in the main directory of this source tree, or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * OpenIB.org BSD license below:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Redistribution and use in source and binary forms, with or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * without modification, are permitted provided that the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * conditions are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * - Redistributions of source code must retain the above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * copyright notice, this list of conditions and the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * - Redistributions in binary form must reproduce the above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * copyright notice, this list of conditions and the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * disclaimer in the documentation and/or other materials
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * provided with the distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * SOFTWARE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/rculist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/llist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include "rds_single_path.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include "ib_mr.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include "rds.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct workqueue_struct *rds_ib_mr_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct rds_ib_dereg_odp_mr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct work_struct work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct ib_mr *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static void rds_ib_odp_mr_worker(struct work_struct *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static struct rds_ib_device *rds_ib_get_device(__be32 ipaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct rds_ib_device *rds_ibdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct rds_ib_ipaddr *i_ipaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) list_for_each_entry_rcu(rds_ibdev, &rds_ib_devices, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) list_for_each_entry_rcu(i_ipaddr, &rds_ibdev->ipaddr_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) if (i_ipaddr->ipaddr == ipaddr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) refcount_inc(&rds_ibdev->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) return rds_ibdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) static int rds_ib_add_ipaddr(struct rds_ib_device *rds_ibdev, __be32 ipaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) struct rds_ib_ipaddr *i_ipaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) i_ipaddr = kmalloc(sizeof *i_ipaddr, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (!i_ipaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) i_ipaddr->ipaddr = ipaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) spin_lock_irq(&rds_ibdev->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) list_add_tail_rcu(&i_ipaddr->list, &rds_ibdev->ipaddr_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) spin_unlock_irq(&rds_ibdev->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static void rds_ib_remove_ipaddr(struct rds_ib_device *rds_ibdev, __be32 ipaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct rds_ib_ipaddr *i_ipaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct rds_ib_ipaddr *to_free = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) spin_lock_irq(&rds_ibdev->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) list_for_each_entry_rcu(i_ipaddr, &rds_ibdev->ipaddr_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (i_ipaddr->ipaddr == ipaddr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) list_del_rcu(&i_ipaddr->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) to_free = i_ipaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) spin_unlock_irq(&rds_ibdev->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (to_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) kfree_rcu(to_free, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) int rds_ib_update_ipaddr(struct rds_ib_device *rds_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct in6_addr *ipaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) struct rds_ib_device *rds_ibdev_old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) rds_ibdev_old = rds_ib_get_device(ipaddr->s6_addr32[3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (!rds_ibdev_old)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return rds_ib_add_ipaddr(rds_ibdev, ipaddr->s6_addr32[3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if (rds_ibdev_old != rds_ibdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) rds_ib_remove_ipaddr(rds_ibdev_old, ipaddr->s6_addr32[3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) rds_ib_dev_put(rds_ibdev_old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return rds_ib_add_ipaddr(rds_ibdev, ipaddr->s6_addr32[3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) rds_ib_dev_put(rds_ibdev_old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) void rds_ib_add_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) struct rds_ib_connection *ic = conn->c_transport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) /* conn was previously on the nodev_conns_list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) spin_lock_irq(&ib_nodev_conns_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) BUG_ON(list_empty(&ib_nodev_conns));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) BUG_ON(list_empty(&ic->ib_node));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) list_del(&ic->ib_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) spin_lock(&rds_ibdev->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) list_add_tail(&ic->ib_node, &rds_ibdev->conn_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) spin_unlock(&rds_ibdev->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) spin_unlock_irq(&ib_nodev_conns_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) ic->rds_ibdev = rds_ibdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) refcount_inc(&rds_ibdev->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) void rds_ib_remove_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct rds_ib_connection *ic = conn->c_transport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /* place conn on nodev_conns_list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) spin_lock(&ib_nodev_conns_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) spin_lock_irq(&rds_ibdev->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) BUG_ON(list_empty(&ic->ib_node));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) list_del(&ic->ib_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) spin_unlock_irq(&rds_ibdev->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) list_add_tail(&ic->ib_node, &ib_nodev_conns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) spin_unlock(&ib_nodev_conns_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) ic->rds_ibdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) rds_ib_dev_put(rds_ibdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) void rds_ib_destroy_nodev_conns(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct rds_ib_connection *ic, *_ic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) LIST_HEAD(tmp_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) /* avoid calling conn_destroy with irqs off */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) spin_lock_irq(&ib_nodev_conns_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) list_splice(&ib_nodev_conns, &tmp_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) spin_unlock_irq(&ib_nodev_conns_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) list_for_each_entry_safe(ic, _ic, &tmp_list, ib_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) rds_conn_destroy(ic->conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) void rds_ib_get_mr_info(struct rds_ib_device *rds_ibdev, struct rds_info_rdma_connection *iinfo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) struct rds_ib_mr_pool *pool_1m = rds_ibdev->mr_1m_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) iinfo->rdma_mr_max = pool_1m->max_items;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) iinfo->rdma_mr_size = pool_1m->max_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) #if IS_ENABLED(CONFIG_IPV6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) void rds6_ib_get_mr_info(struct rds_ib_device *rds_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct rds6_info_rdma_connection *iinfo6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) struct rds_ib_mr_pool *pool_1m = rds_ibdev->mr_1m_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) iinfo6->rdma_mr_max = pool_1m->max_items;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) iinfo6->rdma_mr_size = pool_1m->max_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) struct rds_ib_mr *ibmr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) struct llist_node *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) spin_lock_irqsave(&pool->clean_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) ret = llist_del_first(&pool->clean_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) spin_unlock_irqrestore(&pool->clean_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) ibmr = llist_entry(ret, struct rds_ib_mr, llnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if (pool->pool_type == RDS_IB_MR_8K_POOL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) rds_ib_stats_inc(s_ib_rdma_mr_8k_reused);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) rds_ib_stats_inc(s_ib_rdma_mr_1m_reused);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) return ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) void rds_ib_sync_mr(void *trans_private, int direction)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) struct rds_ib_mr *ibmr = trans_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct rds_ib_device *rds_ibdev = ibmr->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (ibmr->odp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) switch (direction) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) case DMA_FROM_DEVICE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) ib_dma_sync_sg_for_cpu(rds_ibdev->dev, ibmr->sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) ibmr->sg_dma_len, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) case DMA_TO_DEVICE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) ib_dma_sync_sg_for_device(rds_ibdev->dev, ibmr->sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) ibmr->sg_dma_len, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) void __rds_ib_teardown_mr(struct rds_ib_mr *ibmr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) struct rds_ib_device *rds_ibdev = ibmr->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) if (ibmr->sg_dma_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) ib_dma_unmap_sg(rds_ibdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) ibmr->sg, ibmr->sg_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) ibmr->sg_dma_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /* Release the s/g list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (ibmr->sg_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) for (i = 0; i < ibmr->sg_len; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct page *page = sg_page(&ibmr->sg[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) /* FIXME we need a way to tell a r/w MR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * from a r/o MR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) WARN_ON(!page->mapping && irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) kfree(ibmr->sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) ibmr->sg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) ibmr->sg_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) void rds_ib_teardown_mr(struct rds_ib_mr *ibmr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) unsigned int pinned = ibmr->sg_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) __rds_ib_teardown_mr(ibmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (pinned) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct rds_ib_mr_pool *pool = ibmr->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) atomic_sub(pinned, &pool->free_pinned);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static inline unsigned int rds_ib_flush_goal(struct rds_ib_mr_pool *pool, int free_all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) unsigned int item_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) item_count = atomic_read(&pool->item_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (free_all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return item_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * given an llist of mrs, put them all into the list_head for more processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) static unsigned int llist_append_to_list(struct llist_head *llist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) struct list_head *list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) struct rds_ib_mr *ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) struct llist_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) struct llist_node *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) unsigned int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) node = llist_del_all(llist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) while (node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) next = node->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) ibmr = llist_entry(node, struct rds_ib_mr, llnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) list_add_tail(&ibmr->unmap_list, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) node = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * this takes a list head of mrs and turns it into linked llist nodes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * of clusters. Each cluster has linked llist nodes of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * MR_CLUSTER_SIZE mrs that are ready for reuse.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static void list_to_llist_nodes(struct list_head *list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) struct llist_node **nodes_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) struct llist_node **nodes_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) struct rds_ib_mr *ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) struct llist_node *cur = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) struct llist_node **next = nodes_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) list_for_each_entry(ibmr, list, unmap_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) cur = &ibmr->llnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) *next = cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) next = &cur->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) *next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) *nodes_tail = cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * Flush our pool of MRs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * At a minimum, all currently unused MRs are unmapped.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * If the number of MRs allocated exceeds the limit, we also try
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * to free as many MRs as needed to get back to this limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) int free_all, struct rds_ib_mr **ibmr_ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct rds_ib_mr *ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) struct llist_node *clean_nodes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) struct llist_node *clean_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) LIST_HEAD(unmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) unsigned long unpinned = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) unsigned int nfreed = 0, dirty_to_clean = 0, free_goal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (pool->pool_type == RDS_IB_MR_8K_POOL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (ibmr_ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) DEFINE_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) while (!mutex_trylock(&pool->flush_lock)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) ibmr = rds_ib_reuse_mr(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (ibmr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) *ibmr_ret = ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) finish_wait(&pool->flush_wait, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) goto out_nolock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) prepare_to_wait(&pool->flush_wait, &wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) if (llist_empty(&pool->clean_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) ibmr = rds_ib_reuse_mr(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) if (ibmr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) *ibmr_ret = ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) finish_wait(&pool->flush_wait, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) goto out_nolock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) finish_wait(&pool->flush_wait, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) mutex_lock(&pool->flush_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (ibmr_ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) ibmr = rds_ib_reuse_mr(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (ibmr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) *ibmr_ret = ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) /* Get the list of all MRs to be dropped. Ordering matters -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * we want to put drop_list ahead of free_list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) dirty_to_clean = llist_append_to_list(&pool->drop_list, &unmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) dirty_to_clean += llist_append_to_list(&pool->free_list, &unmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (free_all) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) spin_lock_irqsave(&pool->clean_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) llist_append_to_list(&pool->clean_list, &unmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) spin_unlock_irqrestore(&pool->clean_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) free_goal = rds_ib_flush_goal(pool, free_all);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (list_empty(&unmap_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) rds_ib_unreg_frmr(&unmap_list, &nfreed, &unpinned, free_goal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) if (!list_empty(&unmap_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) list_to_llist_nodes(&unmap_list, &clean_nodes, &clean_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (ibmr_ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) *ibmr_ret = llist_entry(clean_nodes, struct rds_ib_mr, llnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) clean_nodes = clean_nodes->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /* more than one entry in llist nodes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (clean_nodes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) spin_lock_irqsave(&pool->clean_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) llist_add_batch(clean_nodes, clean_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) &pool->clean_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) spin_unlock_irqrestore(&pool->clean_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) atomic_sub(unpinned, &pool->free_pinned);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) atomic_sub(dirty_to_clean, &pool->dirty_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) atomic_sub(nfreed, &pool->item_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) mutex_unlock(&pool->flush_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (waitqueue_active(&pool->flush_wait))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) wake_up(&pool->flush_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) out_nolock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct rds_ib_mr *ibmr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) int iter = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) ibmr = rds_ib_reuse_mr(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) if (ibmr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (atomic_inc_return(&pool->item_count) <= pool->max_items)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) atomic_dec(&pool->item_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) if (++iter > 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (pool->pool_type == RDS_IB_MR_8K_POOL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_depleted);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_depleted);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) /* We do have some empty MRs. Flush them out. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (pool->pool_type == RDS_IB_MR_8K_POOL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) rds_ib_flush_mr_pool(pool, 0, &ibmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (ibmr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static void rds_ib_mr_pool_flush_worker(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) struct rds_ib_mr_pool *pool = container_of(work, struct rds_ib_mr_pool, flush_worker.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) rds_ib_flush_mr_pool(pool, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) void rds_ib_free_mr(void *trans_private, int invalidate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) struct rds_ib_mr *ibmr = trans_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) struct rds_ib_mr_pool *pool = ibmr->pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct rds_ib_device *rds_ibdev = ibmr->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) rdsdebug("RDS/IB: free_mr nents %u\n", ibmr->sg_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (ibmr->odp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) /* A MR created and marked as use_once. We use delayed work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * because there is a change that we are in interrupt and can't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) * call to ib_dereg_mr() directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) INIT_DELAYED_WORK(&ibmr->work, rds_ib_odp_mr_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) queue_delayed_work(rds_ib_mr_wq, &ibmr->work, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) /* Return it to the pool's free list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) rds_ib_free_frmr_list(ibmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) atomic_add(ibmr->sg_len, &pool->free_pinned);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) atomic_inc(&pool->dirty_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) /* If we've pinned too many pages, request a flush */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (atomic_read(&pool->free_pinned) >= pool->max_free_pinned ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) atomic_read(&pool->dirty_count) >= pool->max_items / 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) queue_delayed_work(rds_ib_mr_wq, &pool->flush_worker, 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (invalidate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (likely(!in_interrupt())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) rds_ib_flush_mr_pool(pool, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /* We get here if the user created a MR marked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * as use_once and invalidate at the same time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) queue_delayed_work(rds_ib_mr_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) &pool->flush_worker, 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) rds_ib_dev_put(rds_ibdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) void rds_ib_flush_mrs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) struct rds_ib_device *rds_ibdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) down_read(&rds_ib_devices_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) list_for_each_entry(rds_ibdev, &rds_ib_devices, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (rds_ibdev->mr_8k_pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) rds_ib_flush_mr_pool(rds_ibdev->mr_8k_pool, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (rds_ibdev->mr_1m_pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) rds_ib_flush_mr_pool(rds_ibdev->mr_1m_pool, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) up_read(&rds_ib_devices_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) u32 rds_ib_get_lkey(void *trans_private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct rds_ib_mr *ibmr = trans_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) return ibmr->u.mr->lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) struct rds_sock *rs, u32 *key_ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) struct rds_connection *conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) u64 start, u64 length, int need_odp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) struct rds_ib_device *rds_ibdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) struct rds_ib_mr *ibmr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) struct rds_ib_connection *ic = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) rds_ibdev = rds_ib_get_device(rs->rs_bound_addr.s6_addr32[3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (!rds_ibdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (need_odp == ODP_ZEROBASED || need_odp == ODP_VIRTUAL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) u64 virt_addr = need_odp == ODP_ZEROBASED ? 0 : start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) int access_flags =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) IB_ACCESS_ON_DEMAND);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) struct ib_sge sge = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) struct ib_mr *ib_mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) if (!rds_ibdev->odp_capable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) ret = -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) ib_mr = ib_reg_user_mr(rds_ibdev->pd, start, length, virt_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) access_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (IS_ERR(ib_mr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) rdsdebug("rds_ib_get_user_mr returned %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) IS_ERR(ib_mr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) ret = PTR_ERR(ib_mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (key_ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) *key_ret = ib_mr->rkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) ibmr = kzalloc(sizeof(*ibmr), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) if (!ibmr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) ib_dereg_mr(ib_mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) ibmr->u.mr = ib_mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) ibmr->odp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) sge.addr = virt_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) sge.length = length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) sge.lkey = ib_mr->lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) ib_advise_mr(rds_ibdev->pd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) IB_UVERBS_ADVISE_MR_FLAG_FLUSH, &sge, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) return ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) if (conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) ic = conn->c_transport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if (!rds_ibdev->mr_8k_pool || !rds_ibdev->mr_1m_pool) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) ibmr = rds_ib_reg_frmr(rds_ibdev, ic, sg, nents, key_ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) if (IS_ERR(ibmr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) ret = PTR_ERR(ibmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) pr_warn("RDS/IB: rds_ib_get_mr failed (errno=%d)\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) return ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (rds_ibdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) rds_ib_dev_put(rds_ibdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) cancel_delayed_work_sync(&pool->flush_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) rds_ib_flush_mr_pool(pool, 1, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) WARN_ON(atomic_read(&pool->item_count));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) WARN_ON(atomic_read(&pool->free_pinned));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) kfree(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) int pool_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) struct rds_ib_mr_pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) pool = kzalloc(sizeof(*pool), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (!pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) pool->pool_type = pool_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) init_llist_head(&pool->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) init_llist_head(&pool->drop_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) init_llist_head(&pool->clean_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) spin_lock_init(&pool->clean_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) mutex_init(&pool->flush_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) init_waitqueue_head(&pool->flush_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) INIT_DELAYED_WORK(&pool->flush_worker, rds_ib_mr_pool_flush_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) if (pool_type == RDS_IB_MR_1M_POOL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) /* +1 allows for unaligned MRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) pool->max_pages = RDS_MR_1M_MSG_SIZE + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) pool->max_items = rds_ibdev->max_1m_mrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) /* pool_type == RDS_IB_MR_8K_POOL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) pool->max_pages = RDS_MR_8K_MSG_SIZE + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) pool->max_items = rds_ibdev->max_8k_mrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) pool->max_free_pinned = pool->max_items * pool->max_pages / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) pool->max_items_soft = rds_ibdev->max_mrs * 3 / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) return pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) int rds_ib_mr_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) rds_ib_mr_wq = alloc_workqueue("rds_mr_flushd", WQ_MEM_RECLAIM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (!rds_ib_mr_wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) /* By the time this is called all the IB devices should have been torn down and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) * had their pools freed. As each pool is freed its work struct is waited on,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) * so the pool flushing work queue should be idle by the time we get here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) void rds_ib_mr_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) destroy_workqueue(rds_ib_mr_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) static void rds_ib_odp_mr_worker(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) struct rds_ib_mr *ibmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) ibmr = container_of(work, struct rds_ib_mr, work.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) ib_dereg_mr(ibmr->u.mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) kfree(ibmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags