^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright (c) 2006 Oracle. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This software is available to you under a choice of one of two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * licenses. You may choose to be licensed under the terms of the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * General Public License (GPL) Version 2, available from the file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * COPYING in the main directory of this source tree, or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * OpenIB.org BSD license below:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Redistribution and use in source and binary forms, with or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * without modification, are permitted provided that the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * conditions are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * - Redistributions of source code must retain the above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * copyright notice, this list of conditions and the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * - Redistributions in binary form must reproduce the above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * copyright notice, this list of conditions and the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * disclaimer in the documentation and/or other materials
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * provided with the distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * SOFTWARE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include "rds.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include "ib.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * Locking for IB rings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * We assume that allocation is always protected by a mutex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * in the caller (this is a valid assumption for the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * implementation).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * Freeing always happens in an interrupt, and hence only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * races with allocations, but not with other free()s.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * The interaction between allocation and freeing is that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * the alloc code has to determine the number of free entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * To this end, we maintain two counters; an allocation counter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * and a free counter. Both are allowed to run freely, and wrap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * around.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * The number of used entries is always (alloc_ctr - free_ctr) % NR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * The current implementation makes free_ctr atomic. When the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * caller finds an allocation fails, it should set an "alloc fail"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * bit and retry the allocation. The "alloc fail" bit essentially tells
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * the CQ completion handlers to wake it up after freeing some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * more entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * This only happens on shutdown.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) DECLARE_WAIT_QUEUE_HEAD(rds_ib_ring_empty_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) void rds_ib_ring_init(struct rds_ib_work_ring *ring, u32 nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) memset(ring, 0, sizeof(*ring));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) ring->w_nr = nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) rdsdebug("ring %p nr %u\n", ring, ring->w_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static inline u32 __rds_ib_ring_used(struct rds_ib_work_ring *ring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) u32 diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /* This assumes that atomic_t has at least as many bits as u32 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) diff = ring->w_alloc_ctr - (u32) atomic_read(&ring->w_free_ctr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) BUG_ON(diff > ring->w_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) void rds_ib_ring_resize(struct rds_ib_work_ring *ring, u32 nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) /* We only ever get called from the connection setup code,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * prior to creating the QP. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) BUG_ON(__rds_ib_ring_used(ring));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) ring->w_nr = nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static int __rds_ib_ring_empty(struct rds_ib_work_ring *ring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return __rds_ib_ring_used(ring) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) u32 rds_ib_ring_alloc(struct rds_ib_work_ring *ring, u32 val, u32 *pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) u32 ret = 0, avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) avail = ring->w_nr - __rds_ib_ring_used(ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) rdsdebug("ring %p val %u next %u free %u\n", ring, val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) ring->w_alloc_ptr, avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (val && avail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) ret = min(val, avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) *pos = ring->w_alloc_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) ring->w_alloc_ptr = (ring->w_alloc_ptr + ret) % ring->w_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) ring->w_alloc_ctr += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) void rds_ib_ring_free(struct rds_ib_work_ring *ring, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) ring->w_free_ptr = (ring->w_free_ptr + val) % ring->w_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) atomic_add(val, &ring->w_free_ctr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (__rds_ib_ring_empty(ring) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) waitqueue_active(&rds_ib_ring_empty_wait))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) wake_up(&rds_ib_ring_empty_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) void rds_ib_ring_unalloc(struct rds_ib_work_ring *ring, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) ring->w_alloc_ptr = (ring->w_alloc_ptr - val) % ring->w_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) ring->w_alloc_ctr -= val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) int rds_ib_ring_empty(struct rds_ib_work_ring *ring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return __rds_ib_ring_empty(ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) int rds_ib_ring_low(struct rds_ib_work_ring *ring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return __rds_ib_ring_used(ring) <= (ring->w_nr >> 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * returns the oldest alloced ring entry. This will be the next one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * freed. This can't be called if there are none allocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) u32 rds_ib_ring_oldest(struct rds_ib_work_ring *ring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return ring->w_free_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * returns the number of completed work requests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) u32 rds_ib_ring_completed(struct rds_ib_work_ring *ring, u32 wr_id, u32 oldest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) u32 ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (oldest <= (unsigned long long)wr_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) ret = (unsigned long long)wr_id - oldest + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) ret = ring->w_nr - oldest + (unsigned long long)wr_id + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) rdsdebug("ring %p ret %u wr_id %u oldest %u\n", ring, ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) wr_id, oldest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags