^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright (c) 2006, 2019 Oracle and/or its affiliates. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This software is available to you under a choice of one of two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * licenses. You may choose to be licensed under the terms of the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * General Public License (GPL) Version 2, available from the file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * COPYING in the main directory of this source tree, or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * OpenIB.org BSD license below:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Redistribution and use in source and binary forms, with or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * without modification, are permitted provided that the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * conditions are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * - Redistributions of source code must retain the above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * copyright notice, this list of conditions and the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * - Redistributions in binary form must reproduce the above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * copyright notice, this list of conditions and the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * disclaimer in the documentation and/or other materials
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * provided with the distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * SOFTWARE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <rdma/rdma_cm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include "rds_single_path.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include "rds.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include "ib.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static struct kmem_cache *rds_ib_incoming_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static struct kmem_cache *rds_ib_frag_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) static atomic_t rds_ib_allocation = ATOMIC_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) void rds_ib_recv_init_ring(struct rds_ib_connection *ic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct rds_ib_recv_work *recv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct ib_sge *sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) recv->r_ibinc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) recv->r_frag = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) recv->r_wr.next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) recv->r_wr.wr_id = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) recv->r_wr.sg_list = recv->r_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) recv->r_wr.num_sge = RDS_IB_RECV_SGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) sge = &recv->r_sge[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) sge->addr = ic->i_recv_hdrs_dma[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) sge->length = sizeof(struct rds_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) sge->lkey = ic->i_pd->local_dma_lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) sge = &recv->r_sge[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) sge->addr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) sge->length = RDS_FRAG_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) sge->lkey = ic->i_pd->local_dma_lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * The entire 'from' list, including the from element itself, is put on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * to the tail of the 'to' list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static void list_splice_entire_tail(struct list_head *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct list_head *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct list_head *from_last = from->prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) list_splice_tail(from_last, to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) list_add_tail(from_last, to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static void rds_ib_cache_xfer_to_ready(struct rds_ib_refill_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct list_head *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) tmp = xchg(&cache->xfer, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) if (cache->ready)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) list_splice_entire_tail(tmp, cache->ready);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) cache->ready = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static int rds_ib_recv_alloc_cache(struct rds_ib_refill_cache *cache, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) struct rds_ib_cache_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) cache->percpu = alloc_percpu_gfp(struct rds_ib_cache_head, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (!cache->percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) head = per_cpu_ptr(cache->percpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) head->first = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) head->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) cache->xfer = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) cache->ready = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) ret = rds_ib_recv_alloc_cache(&ic->i_cache_incs, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) ret = rds_ib_recv_alloc_cache(&ic->i_cache_frags, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) free_percpu(ic->i_cache_incs.percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void rds_ib_cache_splice_all_lists(struct rds_ib_refill_cache *cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct list_head *caller_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) struct rds_ib_cache_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) head = per_cpu_ptr(cache->percpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) if (head->first) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) list_splice_entire_tail(head->first, caller_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) head->first = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) if (cache->ready) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) list_splice_entire_tail(cache->ready, caller_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) cache->ready = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) void rds_ib_recv_free_caches(struct rds_ib_connection *ic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) struct rds_ib_incoming *inc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct rds_ib_incoming *inc_tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) struct rds_page_frag *frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) struct rds_page_frag *frag_tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) LIST_HEAD(list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) rds_ib_cache_splice_all_lists(&ic->i_cache_incs, &list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) free_percpu(ic->i_cache_incs.percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) list_for_each_entry_safe(inc, inc_tmp, &list, ii_cache_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) list_del(&inc->ii_cache_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) WARN_ON(!list_empty(&inc->ii_frags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) kmem_cache_free(rds_ib_incoming_slab, inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) atomic_dec(&rds_ib_allocation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) free_percpu(ic->i_cache_frags.percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) list_for_each_entry_safe(frag, frag_tmp, &list, f_cache_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) list_del(&frag->f_cache_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) WARN_ON(!list_empty(&frag->f_item));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) kmem_cache_free(rds_ib_frag_slab, frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /* fwd decl */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) static void rds_ib_recv_cache_put(struct list_head *new_item,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct rds_ib_refill_cache *cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) /* Recycle frag and attached recv buffer f_sg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) static void rds_ib_frag_free(struct rds_ib_connection *ic,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct rds_page_frag *frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) rdsdebug("frag %p page %p\n", frag, sg_page(&frag->f_sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) rds_ib_recv_cache_put(&frag->f_cache_entry, &ic->i_cache_frags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) atomic_add(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) rds_ib_stats_add(s_ib_recv_added_to_cache, RDS_FRAG_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) /* Recycle inc after freeing attached frags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) void rds_ib_inc_free(struct rds_incoming *inc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) struct rds_ib_incoming *ibinc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) struct rds_page_frag *frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) struct rds_page_frag *pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) struct rds_ib_connection *ic = inc->i_conn->c_transport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) /* Free attached frags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) list_for_each_entry_safe(frag, pos, &ibinc->ii_frags, f_item) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) list_del_init(&frag->f_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) rds_ib_frag_free(ic, frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) BUG_ON(!list_empty(&ibinc->ii_frags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) rdsdebug("freeing ibinc %p inc %p\n", ibinc, inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) rds_ib_recv_cache_put(&ibinc->ii_cache_entry, &ic->i_cache_incs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static void rds_ib_recv_clear_one(struct rds_ib_connection *ic,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) struct rds_ib_recv_work *recv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (recv->r_ibinc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) rds_inc_put(&recv->r_ibinc->ii_inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) recv->r_ibinc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (recv->r_frag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) rds_ib_frag_free(ic, recv->r_frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) recv->r_frag = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) void rds_ib_recv_clear_ring(struct rds_ib_connection *ic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) for (i = 0; i < ic->i_recv_ring.w_nr; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) rds_ib_recv_clear_one(ic, &ic->i_recvs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) gfp_t slab_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) struct rds_ib_incoming *ibinc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct list_head *cache_item;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) int avail_allocs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) cache_item = rds_ib_recv_cache_get(&ic->i_cache_incs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) if (cache_item) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) ibinc = container_of(cache_item, struct rds_ib_incoming, ii_cache_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) avail_allocs = atomic_add_unless(&rds_ib_allocation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 1, rds_ib_sysctl_max_recv_allocation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (!avail_allocs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) rds_ib_stats_inc(s_ib_rx_alloc_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) ibinc = kmem_cache_alloc(rds_ib_incoming_slab, slab_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) if (!ibinc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) atomic_dec(&rds_ib_allocation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) rds_ib_stats_inc(s_ib_rx_total_incs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) INIT_LIST_HEAD(&ibinc->ii_frags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) rds_inc_init(&ibinc->ii_inc, ic->conn, &ic->conn->c_faddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return ibinc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) gfp_t slab_mask, gfp_t page_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct rds_page_frag *frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) struct list_head *cache_item;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) cache_item = rds_ib_recv_cache_get(&ic->i_cache_frags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) if (cache_item) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) frag = container_of(cache_item, struct rds_page_frag, f_cache_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) atomic_sub(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) rds_ib_stats_add(s_ib_recv_added_to_cache, RDS_FRAG_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) frag = kmem_cache_alloc(rds_ib_frag_slab, slab_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) if (!frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) sg_init_table(&frag->f_sg, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) ret = rds_page_remainder_alloc(&frag->f_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) RDS_FRAG_SIZE, page_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) kmem_cache_free(rds_ib_frag_slab, frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) rds_ib_stats_inc(s_ib_rx_total_frags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) INIT_LIST_HEAD(&frag->f_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) return frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) static int rds_ib_recv_refill_one(struct rds_connection *conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) struct rds_ib_recv_work *recv, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) struct rds_ib_connection *ic = conn->c_transport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) struct ib_sge *sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) gfp_t slab_mask = gfp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) gfp_t page_mask = gfp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (gfp & __GFP_DIRECT_RECLAIM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) slab_mask = GFP_KERNEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) page_mask = GFP_HIGHUSER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (!ic->i_cache_incs.ready)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (!ic->i_cache_frags.ready)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * ibinc was taken from recv if recv contained the start of a message.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * recvs that were continuations will still have this allocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (!recv->r_ibinc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (!recv->r_ibinc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) WARN_ON(recv->r_frag); /* leak! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (!recv->r_frag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 1, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) WARN_ON(ret != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) sge = &recv->r_sge[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) sge->addr = ic->i_recv_hdrs_dma[recv - ic->i_recvs];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) sge->length = sizeof(struct rds_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) sge = &recv->r_sge[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) sge->addr = sg_dma_address(&recv->r_frag->f_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) sge->length = sg_dma_len(&recv->r_frag->f_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static int acquire_refill(struct rds_connection *conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return test_and_set_bit(RDS_RECV_REFILL, &conn->c_flags) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) static void release_refill(struct rds_connection *conn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) clear_bit(RDS_RECV_REFILL, &conn->c_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /* We don't use wait_on_bit()/wake_up_bit() because our waking is in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * hot path and finding waiters is very rare. We don't want to walk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * the system-wide hashed waitqueue buckets in the fast path only to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * almost never find waiters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (waitqueue_active(&conn->c_waitq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) wake_up_all(&conn->c_waitq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * This tries to allocate and post unused work requests after making sure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * they have all the allocations they need to queue received fragments into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * sockets.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct rds_ib_connection *ic = conn->c_transport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) struct rds_ib_recv_work *recv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) unsigned int posted = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) bool can_wait = !!(gfp & __GFP_DIRECT_RECLAIM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) bool must_wake = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) u32 pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) /* the goal here is to just make sure that someone, somewhere
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * is posting buffers. If we can't get the refill lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) * let them do their thing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) if (!acquire_refill(conn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) while ((prefill || rds_conn_up(conn)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (pos >= ic->i_recv_ring.w_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) printk(KERN_NOTICE "Argh - ring alloc returned pos=%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) recv = &ic->i_recvs[pos];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) ret = rds_ib_recv_refill_one(conn, recv, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) must_wake = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) rdsdebug("recv %p ibinc %p page %p addr %lu\n", recv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) recv->r_ibinc, sg_page(&recv->r_frag->f_sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) (long)sg_dma_address(&recv->r_frag->f_sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) /* XXX when can this fail? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) rds_ib_conn_error(conn, "recv post on "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) "%pI6c returned %d, disconnecting and "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) "reconnecting\n", &conn->c_faddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) posted++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) if ((posted > 128 && need_resched()) || posted > 8192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) must_wake = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) /* We're doing flow control - update the window. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) if (ic->i_flowctl && posted)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) rds_ib_advertise_credits(conn, posted);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) rds_ib_ring_unalloc(&ic->i_recv_ring, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) release_refill(conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /* if we're called from the softirq handler, we'll be GFP_NOWAIT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * in this case the ring being low is going to lead to more interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * and we can safely let the softirq code take care of it unless the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * ring is completely empty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * if we're called from krdsd, we'll be GFP_KERNEL. In this case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * we might have raced with the softirq code while we had the refill
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * lock held. Use rds_ib_ring_low() instead of ring_empty to decide
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * if we should requeue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) if (rds_conn_up(conn) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) (must_wake ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) (can_wait && rds_ib_ring_low(&ic->i_recv_ring)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) rds_ib_ring_empty(&ic->i_recv_ring))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) queue_delayed_work(rds_wq, &conn->c_recv_w, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (can_wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * We want to recycle several types of recv allocations, like incs and frags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * To use this, the *_free() function passes in the ptr to a list_head within
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * the recyclee, as well as the cache to put it on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * First, we put the memory on a percpu list. When this reaches a certain size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * We move it to an intermediate non-percpu list in a lockless manner, with some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * xchg/compxchg wizardry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * N.B. Instead of a list_head as the anchor, we use a single pointer, which can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * be NULL and xchg'd. The list is actually empty when the pointer is NULL, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * list_empty() will return true with one element is actually present.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static void rds_ib_recv_cache_put(struct list_head *new_item,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) struct rds_ib_refill_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) struct list_head *old, *chpfirst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) chpfirst = __this_cpu_read(cache->percpu->first);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (!chpfirst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) INIT_LIST_HEAD(new_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) else /* put on front */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) list_add_tail(new_item, chpfirst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) __this_cpu_write(cache->percpu->first, new_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) __this_cpu_inc(cache->percpu->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (__this_cpu_read(cache->percpu->count) < RDS_IB_RECYCLE_BATCH_COUNT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * Return our per-cpu first list to the cache's xfer by atomically
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * grabbing the current xfer list, appending it to our per-cpu list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * and then atomically returning that entire list back to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * cache's xfer list as long as it's still empty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) old = xchg(&cache->xfer, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (old)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) list_splice_entire_tail(old, chpfirst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) old = cmpxchg(&cache->xfer, NULL, chpfirst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) } while (old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) __this_cpu_write(cache->percpu->first, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) __this_cpu_write(cache->percpu->count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) end:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) struct list_head *head = cache->ready;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (!list_empty(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) cache->ready = head->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) list_del_init(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) cache->ready = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) return head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) struct rds_ib_incoming *ibinc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct rds_page_frag *frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) unsigned long to_copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) unsigned long frag_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) int copied = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) u32 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) frag = list_entry(ibinc->ii_frags.next, struct rds_page_frag, f_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) len = be32_to_cpu(inc->i_hdr.h_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) while (iov_iter_count(to) && copied < len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (frag_off == RDS_FRAG_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) frag = list_entry(frag->f_item.next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) struct rds_page_frag, f_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) frag_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) to_copy = min_t(unsigned long, iov_iter_count(to),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) RDS_FRAG_SIZE - frag_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) to_copy = min_t(unsigned long, to_copy, len - copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) /* XXX needs + offset for multiple recvs per page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) rds_stats_add(s_copy_to_user, to_copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) ret = copy_page_to_iter(sg_page(&frag->f_sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) frag->f_sg.offset + frag_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) to_copy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (ret != to_copy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) frag_off += to_copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) copied += to_copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) return copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /* ic starts out kzalloc()ed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) void rds_ib_recv_init_ack(struct rds_ib_connection *ic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) struct ib_send_wr *wr = &ic->i_ack_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) struct ib_sge *sge = &ic->i_ack_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) sge->addr = ic->i_ack_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) sge->length = sizeof(struct rds_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) sge->lkey = ic->i_pd->local_dma_lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) wr->sg_list = sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) wr->num_sge = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) wr->opcode = IB_WR_SEND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) wr->wr_id = RDS_IB_ACK_WR_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) wr->send_flags = IB_SEND_SIGNALED | IB_SEND_SOLICITED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) * You'd think that with reliable IB connections you wouldn't need to ack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * messages that have been received. The problem is that IB hardware generates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * an ack message before it has DMAed the message into memory. This creates a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * potential message loss if the HCA is disabled for any reason between when it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * sends the ack and before the message is DMAed and processed. This is only a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * potential issue if another HCA is available for fail-over.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) * When the remote host receives our ack they'll free the sent message from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) * their send queue. To decrease the latency of this we always send an ack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) * immediately after we've received messages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) * For simplicity, we only have one ack in flight at a time. This puts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) * pressure on senders to have deep enough send queues to absorb the latency of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) * a single ack frame being in flight. This might not be good enough.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) * This is implemented by have a long-lived send_wr and sge which point to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) * statically allocated ack frame. This ack wr does not fall under the ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) * accounting that the tx and rx wrs do. The QP attribute specifically makes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) * room for it beyond the ring size. Send completion notices its special
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) * wr_id and avoids working with the ring in that case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) #ifndef KERNEL_HAS_ATOMIC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) spin_lock_irqsave(&ic->i_ack_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) ic->i_ack_next = seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (ack_required)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) spin_unlock_irqrestore(&ic->i_ack_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) static u64 rds_ib_get_ack(struct rds_ib_connection *ic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) u64 seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) spin_lock_irqsave(&ic->i_ack_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) seq = ic->i_ack_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) spin_unlock_irqrestore(&ic->i_ack_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) return seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) atomic64_set(&ic->i_ack_next, seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) if (ack_required) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) static u64 rds_ib_get_ack(struct rds_ib_connection *ic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) return atomic64_read(&ic->i_ack_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) static void rds_ib_send_ack(struct rds_ib_connection *ic, unsigned int adv_credits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) struct rds_header *hdr = ic->i_ack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) u64 seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) seq = rds_ib_get_ack(ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev, ic->i_ack_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) sizeof(*hdr), DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) rds_message_populate_header(hdr, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) hdr->h_ack = cpu_to_be64(seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) hdr->h_credit = adv_credits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) rds_message_make_checksum(hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) ib_dma_sync_single_for_device(ic->rds_ibdev->dev, ic->i_ack_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) sizeof(*hdr), DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) ic->i_ack_queued = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (unlikely(ret)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) /* Failed to send. Release the WR, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * force another ACK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) rds_ib_stats_inc(s_ib_ack_send_failure);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) rds_ib_conn_error(ic->conn, "sending ack failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) rds_ib_stats_inc(s_ib_ack_sent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * There are 3 ways of getting acknowledgements to the peer:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * 1. We call rds_ib_attempt_ack from the recv completion handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) * to send an ACK-only frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * However, there can be only one such frame in the send queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * at any time, so we may have to postpone it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * 2. When another (data) packet is transmitted while there's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * an ACK in the queue, we piggyback the ACK sequence number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * on the data packet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * 3. If the ACK WR is done sending, we get called from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * send queue completion handler, and check whether there's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * another ACK pending (postponed because the WR was on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * queue). If so, we transmit it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * We maintain 2 variables:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) * - i_ack_flags, which keeps track of whether the ACK WR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * is currently in the send queue or not (IB_ACK_IN_FLIGHT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) * - i_ack_next, which is the last sequence number we received
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) * Potentially, send queue and receive queue handlers can run concurrently.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) * It would be nice to not have to use a spinlock to synchronize things,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) * but the one problem that rules this out is that 64bit updates are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) * not atomic on all platforms. Things would be a lot simpler if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * we had atomic64 or maybe cmpxchg64 everywhere.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) * Reconnecting complicates this picture just slightly. When we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) * reconnect, we may be seeing duplicate packets. The peer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * is retransmitting them, because it hasn't seen an ACK for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * them. It is important that we ACK these.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * ACK mitigation adds a header flag "ACK_REQUIRED"; any packet with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) * this flag set *MUST* be acknowledged immediately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) * When we get here, we're called from the recv queue handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) * Check whether we ought to transmit an ACK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) void rds_ib_attempt_ack(struct rds_ib_connection *ic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) unsigned int adv_credits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) rds_ib_stats_inc(s_ib_ack_send_delayed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) /* Can we get a send credit? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) rds_ib_stats_inc(s_ib_tx_throttle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) rds_ib_send_ack(ic, adv_credits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * We get here from the send completion handler, when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * adapter tells us the ACK frame was sent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) void rds_ib_ack_send_complete(struct rds_ib_connection *ic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) rds_ib_attempt_ack(ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * This is called by the regular xmit code when it wants to piggyback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * an ACK on an outgoing frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) rds_ib_stats_inc(s_ib_ack_send_piggybacked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) return rds_ib_get_ack(ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * It's kind of lame that we're copying from the posted receive pages into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * long-lived bitmaps. We could have posted the bitmaps and rdma written into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) * them. But receiving new congestion bitmaps should be a *rare* event, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) * hopefully we won't need to invest that complexity in making it more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * efficient. By copying we can share a simpler core with TCP which has to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * copy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) static void rds_ib_cong_recv(struct rds_connection *conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) struct rds_ib_incoming *ibinc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) struct rds_cong_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) unsigned int map_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) unsigned int map_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) struct rds_page_frag *frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) unsigned long frag_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) unsigned long to_copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) unsigned long copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) __le64 uncongested = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) void *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) /* catch completely corrupt packets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) if (be32_to_cpu(ibinc->ii_inc.i_hdr.h_len) != RDS_CONG_MAP_BYTES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) map = conn->c_fcong;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) map_page = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) map_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) frag = list_entry(ibinc->ii_frags.next, struct rds_page_frag, f_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) frag_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) copied = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) while (copied < RDS_CONG_MAP_BYTES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) __le64 *src, *dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) unsigned int k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) to_copy = min(RDS_FRAG_SIZE - frag_off, PAGE_SIZE - map_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) BUG_ON(to_copy & 7); /* Must be 64bit aligned. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) addr = kmap_atomic(sg_page(&frag->f_sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) src = addr + frag->f_sg.offset + frag_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) dst = (void *)map->m_page_addrs[map_page] + map_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) for (k = 0; k < to_copy; k += 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) /* Record ports that became uncongested, ie
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) * bits that changed from 0 to 1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) uncongested |= ~(*src) & *dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) *dst++ = *src++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) kunmap_atomic(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) copied += to_copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) map_off += to_copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) if (map_off == PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) map_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) map_page++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) frag_off += to_copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) if (frag_off == RDS_FRAG_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) frag = list_entry(frag->f_item.next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) struct rds_page_frag, f_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) frag_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) /* the congestion map is in little endian order */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) rds_cong_map_updated(map, le64_to_cpu(uncongested));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) static void rds_ib_process_recv(struct rds_connection *conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) struct rds_ib_recv_work *recv, u32 data_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) struct rds_ib_ack_state *state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) struct rds_ib_connection *ic = conn->c_transport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) struct rds_ib_incoming *ibinc = ic->i_ibinc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) struct rds_header *ihdr, *hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) dma_addr_t dma_addr = ic->i_recv_hdrs_dma[recv - ic->i_recvs];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) /* XXX shut down the connection if port 0,0 are seen? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) rdsdebug("ic %p ibinc %p recv %p byte len %u\n", ic, ibinc, recv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) data_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) if (data_len < sizeof(struct rds_header)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) rds_ib_conn_error(conn, "incoming message "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) "from %pI6c didn't include a "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) "header, disconnecting and "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) "reconnecting\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) &conn->c_faddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) data_len -= sizeof(struct rds_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) ihdr = ic->i_recv_hdrs[recv - ic->i_recvs];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev, dma_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) sizeof(*ihdr), DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) /* Validate the checksum. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) if (!rds_message_verify_checksum(ihdr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) rds_ib_conn_error(conn, "incoming message "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) "from %pI6c has corrupted header - "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) "forcing a reconnect\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) &conn->c_faddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) rds_stats_inc(s_recv_drop_bad_checksum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) /* Process the ACK sequence which comes with every packet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) state->ack_recv = be64_to_cpu(ihdr->h_ack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) state->ack_recv_valid = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) /* Process the credits update if there was one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if (ihdr->h_credit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) rds_ib_send_add_credits(conn, ihdr->h_credit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) if (ihdr->h_sport == 0 && ihdr->h_dport == 0 && data_len == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) /* This is an ACK-only packet. The fact that it gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * special treatment here is that historically, ACKs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * were rather special beasts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) rds_ib_stats_inc(s_ib_ack_received);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) * Usually the frags make their way on to incs and are then freed as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) * the inc is freed. We don't go that route, so we have to drop the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) * page ref ourselves. We can't just leave the page on the recv
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) * because that confuses the dma mapping of pages and each recv's use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * of a partial page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) * FIXME: Fold this into the code path below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) rds_ib_frag_free(ic, recv->r_frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) recv->r_frag = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * If we don't already have an inc on the connection then this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * fragment has a header and starts a message.. copy its header
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) * into the inc and save the inc so we can hang upcoming fragments
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * off its list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (!ibinc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) ibinc = recv->r_ibinc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) recv->r_ibinc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) ic->i_ibinc = ibinc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) hdr = &ibinc->ii_inc.i_hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) ibinc->ii_inc.i_rx_lat_trace[RDS_MSG_RX_HDR] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) local_clock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) memcpy(hdr, ihdr, sizeof(*hdr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) ic->i_recv_data_rem = be32_to_cpu(hdr->h_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) ibinc->ii_inc.i_rx_lat_trace[RDS_MSG_RX_START] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) local_clock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) rdsdebug("ic %p ibinc %p rem %u flag 0x%x\n", ic, ibinc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) ic->i_recv_data_rem, hdr->h_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) hdr = &ibinc->ii_inc.i_hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) /* We can't just use memcmp here; fragments of a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) * single message may carry different ACKs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) if (hdr->h_sequence != ihdr->h_sequence ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) hdr->h_len != ihdr->h_len ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) hdr->h_sport != ihdr->h_sport ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) hdr->h_dport != ihdr->h_dport) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) rds_ib_conn_error(conn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) "fragment header mismatch; forcing reconnect\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) list_add_tail(&recv->r_frag->f_item, &ibinc->ii_frags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) recv->r_frag = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) if (ic->i_recv_data_rem > RDS_FRAG_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) ic->i_recv_data_rem -= RDS_FRAG_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) ic->i_recv_data_rem = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) ic->i_ibinc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) if (ibinc->ii_inc.i_hdr.h_flags == RDS_FLAG_CONG_BITMAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) rds_ib_cong_recv(conn, ibinc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) rds_recv_incoming(conn, &conn->c_faddr, &conn->c_laddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) &ibinc->ii_inc, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) state->ack_next = be64_to_cpu(hdr->h_sequence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) state->ack_next_valid = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) /* Evaluate the ACK_REQUIRED flag *after* we received
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) * the complete frame, and after bumping the next_rx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) * sequence. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) if (hdr->h_flags & RDS_FLAG_ACK_REQUIRED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) rds_stats_inc(s_recv_ack_required);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) state->ack_required = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) rds_inc_put(&ibinc->ii_inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) ib_dma_sync_single_for_device(ic->rds_ibdev->dev, dma_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) sizeof(*ihdr), DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) struct ib_wc *wc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) struct rds_ib_ack_state *state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) struct rds_connection *conn = ic->conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) struct rds_ib_recv_work *recv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) rdsdebug("wc wr_id 0x%llx status %u (%s) byte_len %u imm_data %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) (unsigned long long)wc->wr_id, wc->status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) ib_wc_status_msg(wc->status), wc->byte_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) be32_to_cpu(wc->ex.imm_data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) rds_ib_stats_inc(s_ib_rx_cq_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) recv = &ic->i_recvs[rds_ib_ring_oldest(&ic->i_recv_ring)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) /* Also process recvs in connecting state because it is possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) * to get a recv completion _before_ the rdmacm ESTABLISHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) * event is processed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) if (wc->status == IB_WC_SUCCESS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) rds_ib_process_recv(conn, recv, wc->byte_len, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) /* We expect errors as the qp is drained during shutdown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) if (rds_conn_up(conn) || rds_conn_connecting(conn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) rds_ib_conn_error(conn, "recv completion on <%pI6c,%pI6c, %d> had status %u (%s), vendor err 0x%x, disconnecting and reconnecting\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) &conn->c_laddr, &conn->c_faddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) conn->c_tos, wc->status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) ib_wc_status_msg(wc->status),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) wc->vendor_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) /* rds_ib_process_recv() doesn't always consume the frag, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) * we might not have called it at all if the wc didn't indicate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) * success. We already unmapped the frag's pages, though, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) * the following rds_ib_ring_free() call tells the refill path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) * that it will not find an allocated frag here. Make sure we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) * keep that promise by freeing a frag that's still on the ring.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) if (recv->r_frag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) rds_ib_frag_free(ic, recv->r_frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) recv->r_frag = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) rds_ib_ring_free(&ic->i_recv_ring, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) /* If we ever end up with a really empty receive ring, we're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) * in deep trouble, as the sender will definitely see RNR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) * timeouts. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if (rds_ib_ring_empty(&ic->i_recv_ring))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) rds_ib_stats_inc(s_ib_rx_ring_empty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) if (rds_ib_ring_low(&ic->i_recv_ring)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) rds_ib_recv_refill(conn, 0, GFP_NOWAIT | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) rds_ib_stats_inc(s_ib_rx_refill_from_cq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) int rds_ib_recv_path(struct rds_conn_path *cp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) struct rds_connection *conn = cp->cp_conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) struct rds_ib_connection *ic = conn->c_transport_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) rdsdebug("conn %p\n", conn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) if (rds_conn_up(conn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) rds_ib_attempt_ack(ic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) rds_ib_recv_refill(conn, 0, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) rds_ib_stats_inc(s_ib_rx_refill_from_thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) int rds_ib_recv_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) struct sysinfo si;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) /* Default to 30% of all available RAM for recv memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) si_meminfo(&si);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) rds_ib_sysctl_max_recv_allocation = si.totalram / 3 * PAGE_SIZE / RDS_FRAG_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) rds_ib_incoming_slab =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) kmem_cache_create_usercopy("rds_ib_incoming",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) sizeof(struct rds_ib_incoming),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) 0, SLAB_HWCACHE_ALIGN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) offsetof(struct rds_ib_incoming,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) ii_inc.i_usercopy),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) sizeof(struct rds_inc_usercopy),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) if (!rds_ib_incoming_slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) rds_ib_frag_slab = kmem_cache_create("rds_ib_frag",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) sizeof(struct rds_page_frag),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) 0, SLAB_HWCACHE_ALIGN, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) if (!rds_ib_frag_slab) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) kmem_cache_destroy(rds_ib_incoming_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) rds_ib_incoming_slab = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) void rds_ib_recv_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) WARN_ON(atomic_read(&rds_ib_allocation));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) kmem_cache_destroy(rds_ib_incoming_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) kmem_cache_destroy(rds_ib_frag_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags