^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright(c) 2016 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #ifndef DEF_RDMAVT_INCMR_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #define DEF_RDMAVT_INCMR_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * For Memory Regions. This stuff should probably be moved into rdmavt/mr.h once
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * drivers no longer need access to the MR directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/percpu-refcount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * A segment is a linear region of low physical memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * Used by the verbs layer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) struct rvt_seg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) void *vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) size_t length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) /* The number of rvt_segs that fit in a page. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define RVT_SEGSZ (PAGE_SIZE / sizeof(struct rvt_seg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) struct rvt_segarray {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct rvt_seg segs[RVT_SEGSZ];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct rvt_mregion {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) struct ib_pd *pd; /* shares refcnt of ibmr.pd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) u64 user_base; /* User's address for this region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) u64 iova; /* IB start address of this region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) size_t length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) u32 lkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) u32 offset; /* offset (bytes) to start of region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) int access_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) u32 max_segs; /* number of rvt_segs in all the arrays */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) u32 mapsz; /* size of the map array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) atomic_t lkey_invalid; /* true if current lkey is invalid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) u8 page_shift; /* 0 - non unform/non powerof2 sizes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) u8 lkey_published; /* in global table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct percpu_ref refcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct completion comp; /* complete when refcount goes to zero */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) struct rvt_segarray *map[]; /* the segments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define RVT_MAX_LKEY_TABLE_BITS 23
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct rvt_lkey_table {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /* read mostly fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) u32 max; /* size of the table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) u32 shift; /* lkey/rkey shift */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct rvt_mregion __rcu **table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /* writeable fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) /* protect changes in this struct */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) spinlock_t lock ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) u32 next; /* next unused index (speeds search) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) u32 gen; /* generation count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * These keep track of the copy progress within a memory region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * Used by the verbs layer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct rvt_sge {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct rvt_mregion *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) void *vaddr; /* kernel virtual address of segment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) u32 sge_length; /* length of the SGE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) u32 length; /* remaining length of the segment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) u16 m; /* current index: mr->map[m] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) u16 n; /* current index: mr->map[m]->segs[n] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct rvt_sge_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) struct rvt_sge *sg_list; /* next SGE to be used if any */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) struct rvt_sge sge; /* progress state for the current SGE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) u32 total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) u8 num_sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static inline void rvt_put_mr(struct rvt_mregion *mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) percpu_ref_put(&mr->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static inline void rvt_get_mr(struct rvt_mregion *mr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) percpu_ref_get(&mr->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static inline void rvt_put_ss(struct rvt_sge_state *ss)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) while (ss->num_sge) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) rvt_put_mr(ss->sge.mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) if (--ss->num_sge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) ss->sge = *ss->sg_list++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static inline u32 rvt_get_sge_length(struct rvt_sge *sge, u32 length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) u32 len = sge->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (len > length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) len = length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (len > sge->sge_length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) len = sge->sge_length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static inline void rvt_update_sge(struct rvt_sge_state *ss, u32 length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) bool release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct rvt_sge *sge = &ss->sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) sge->vaddr += length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) sge->length -= length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) sge->sge_length -= length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (sge->sge_length == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) rvt_put_mr(sge->mr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (--ss->num_sge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) *sge = *ss->sg_list++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) } else if (sge->length == 0 && sge->mr->lkey) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (++sge->n >= RVT_SEGSZ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (++sge->m >= sge->mr->mapsz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) sge->n = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) static inline void rvt_skip_sge(struct rvt_sge_state *ss, u32 length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) bool release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct rvt_sge *sge = &ss->sge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) while (length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) u32 len = rvt_get_sge_length(sge, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) WARN_ON_ONCE(len == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) rvt_update_sge(ss, len, release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) length -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) bool rvt_ss_has_lkey(struct rvt_sge_state *ss, u32 lkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) bool rvt_mr_has_lkey(struct rvt_mregion *mr, u32 lkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #endif /* DEF_RDMAVT_INCMRH */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags