^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Copyright (c) 2016 Oracle. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This software is available to you under a choice of one of two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * licenses. You may choose to be licensed under the terms of the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * General Public License (GPL) Version 2, available from the file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * COPYING in the main directory of this source tree, or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * OpenIB.org BSD license below:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Redistribution and use in source and binary forms, with or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * without modification, are permitted provided that the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * conditions are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * - Redistributions of source code must retain the above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * copyright notice, this list of conditions and the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * - Redistributions in binary form must reproduce the above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * copyright notice, this list of conditions and the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * disclaimer in the documentation and/or other materials
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * provided with the distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * SOFTWARE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #ifndef _RDS_IB_MR_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define _RDS_IB_MR_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include "rds.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include "ib.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define RDS_MR_1M_POOL_SIZE (8192 / 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define RDS_MR_1M_MSG_SIZE 256
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define RDS_MR_8K_MSG_SIZE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define RDS_MR_8K_SCALE (256 / (RDS_MR_8K_MSG_SIZE + 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define RDS_MR_8K_POOL_SIZE (RDS_MR_8K_SCALE * (8192 / 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) enum rds_ib_fr_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) FRMR_IS_FREE, /* mr invalidated & ready for use */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) FRMR_IS_INUSE, /* mr is in use or used & can be invalidated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) FRMR_IS_STALE, /* Stale MR and needs to be dropped */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct rds_ib_frmr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct ib_mr *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) enum rds_ib_fr_state fr_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) bool fr_inv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) wait_queue_head_t fr_inv_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) bool fr_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) wait_queue_head_t fr_reg_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) struct ib_send_wr fr_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) unsigned int dma_npages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) unsigned int sg_byte_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) /* This is stored as mr->r_trans_private. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct rds_ib_mr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct delayed_work work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct rds_ib_device *device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct rds_ib_mr_pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct rds_ib_connection *ic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct llist_node llnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* unmap_list is for freeing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct list_head unmap_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) unsigned int remap_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) unsigned int sg_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) int sg_dma_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) u8 odp:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct rds_ib_frmr frmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) struct ib_mr *mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) } u;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /* Our own little MR pool */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct rds_ib_mr_pool {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) unsigned int pool_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct mutex flush_lock; /* serialize fmr invalidate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct delayed_work flush_worker; /* flush worker */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) atomic_t item_count; /* total # of MRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) atomic_t dirty_count; /* # dirty of MRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct llist_head drop_list; /* MRs not reached max_maps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct llist_head free_list; /* unused MRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) struct llist_head clean_list; /* unused & unmapped MRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) wait_queue_head_t flush_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) spinlock_t clean_lock; /* "clean_list" concurrency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) atomic_t free_pinned; /* memory pinned by free MRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) unsigned long max_items;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) unsigned long max_items_soft;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) unsigned long max_free_pinned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) unsigned int max_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) extern struct workqueue_struct *rds_ib_mr_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) extern bool prefer_frmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) int npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) void rds_ib_get_mr_info(struct rds_ib_device *rds_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct rds_info_rdma_connection *iinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) void rds6_ib_get_mr_info(struct rds_ib_device *rds_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct rds6_info_rdma_connection *iinfo6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) struct rds_sock *rs, u32 *key_ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) struct rds_connection *conn, u64 start, u64 length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) int need_odp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) void rds_ib_sync_mr(void *trans_private, int dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) void rds_ib_free_mr(void *trans_private, int invalidate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) void rds_ib_flush_mrs(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) int rds_ib_mr_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) void rds_ib_mr_exit(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) u32 rds_ib_get_lkey(void *trans_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) void __rds_ib_teardown_mr(struct rds_ib_mr *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) void rds_ib_teardown_mr(struct rds_ib_mr *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *, int, struct rds_ib_mr **);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct rds_ib_mr *rds_ib_reg_frmr(struct rds_ib_device *rds_ibdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct rds_ib_connection *ic,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) struct scatterlist *sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) unsigned long nents, u32 *key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) void rds_ib_unreg_frmr(struct list_head *list, unsigned int *nfreed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) unsigned long *unpinned, unsigned int goal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) void rds_ib_free_frmr_list(struct rds_ib_mr *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags