^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Copyright(c) 2020 Intel Corporation. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #ifndef XSK_BUFF_POOL_H_
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #define XSK_BUFF_POOL_H_
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/if_xdp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <net/xdp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) struct xsk_buff_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) struct xdp_rxq_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) struct xsk_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) struct xdp_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) struct xdp_umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) struct xdp_sock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) struct device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) struct page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) struct xdp_buff_xsk {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) struct xdp_buff xdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) dma_addr_t dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) dma_addr_t frame_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) struct xsk_buff_pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) bool unaligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) u64 orig_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct list_head free_list_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct xsk_dma_map {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) dma_addr_t *dma_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) struct net_device *netdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) refcount_t users;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) struct list_head list; /* Protected by the RTNL_LOCK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) u32 dma_pages_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) bool dma_need_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) struct xsk_buff_pool {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* Members only used in the control path first. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct net_device *netdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct list_head xsk_tx_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /* Protects modifications to the xsk_tx_list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) spinlock_t xsk_tx_list_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) refcount_t users;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct xdp_umem *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct work_struct work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct list_head free_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) u32 heads_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) u16 queue_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /* Data path members as close to free_heads at the end as possible. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct xsk_queue *fq ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct xsk_queue *cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /* For performance reasons, each buff pool has its own array of dma_pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * even when they are identical.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) dma_addr_t *dma_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct xdp_buff_xsk *heads;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) u64 chunk_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) u64 addrs_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) u32 free_list_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) u32 dma_pages_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) u32 free_heads_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) u32 headroom;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) u32 chunk_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) u32 frame_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) u8 cached_need_wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) bool uses_need_wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) bool dma_need_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) bool unaligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) void *addrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /* Mutual exclusion of the completion ring in the SKB mode. Two cases to protect:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * NAPI TX thread and sendmsg error paths in the SKB destructor callback and when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * sockets share a single cq when the same netdev and queue id is shared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) spinlock_t cq_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) struct xdp_buff_xsk *free_heads[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /* AF_XDP core. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) struct xdp_umem *umem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) u16 queue_id, u16 flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct net_device *dev, u16 queue_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) void xp_destroy(struct xsk_buff_pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) void xp_release(struct xdp_buff_xsk *xskb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) void xp_get_pool(struct xsk_buff_pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) bool xp_put_pool(struct xsk_buff_pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) void xp_clear_dev(struct xsk_buff_pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /* AF_XDP, and XDP core. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) void xp_free(struct xdp_buff_xsk *xskb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) /* AF_XDP ZC drivers, via xdp_sock_buff.h */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) unsigned long attrs, struct page **pages, u32 nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return xskb->dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static inline dma_addr_t xp_get_frame_dma(struct xdp_buff_xsk *xskb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return xskb->frame_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) static inline void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) xp_dma_sync_for_cpu_slow(xskb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) size_t size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) dma_addr_t dma, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (!pool->dma_need_sync)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) xp_dma_sync_for_device_slow(pool, dma, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /* Masks for xdp_umem_page flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * The low 12-bits of the addr will be 0 since this is the page address, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * can use them for flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define XSK_NEXT_PG_CONTIG_SHIFT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) u64 addr, u32 len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) bool cross_pg = (addr & (PAGE_SIZE - 1)) + len > PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (likely(!cross_pg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) if (pool->dma_pages_cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) return !(pool->dma_pages[addr >> PAGE_SHIFT] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) XSK_NEXT_PG_CONTIG_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /* skb path */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) return addr + len > pool->addrs_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static inline u64 xp_aligned_extract_addr(struct xsk_buff_pool *pool, u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) return addr & pool->chunk_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static inline u64 xp_unaligned_extract_addr(u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return addr & XSK_UNALIGNED_BUF_ADDR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static inline u64 xp_unaligned_extract_offset(u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) static inline u64 xp_unaligned_add_offset_to_addr(u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) return xp_unaligned_extract_addr(addr) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) xp_unaligned_extract_offset(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #endif /* XSK_BUFF_POOL_H_ */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags