^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * AF_XDP user-space access library.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright(c) 2018 - 2019 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Author(s): Magnus Karlsson <magnus.karlsson@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <stdlib.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <unistd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <arpa/inet.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/barrier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/ethtool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/filter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/if_ether.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/if_packet.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/if_xdp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/sockios.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <net/if.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <sys/ioctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <sys/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <sys/socket.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <sys/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include "bpf.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include "libbpf.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include "libbpf_internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include "xsk.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #ifndef SOL_XDP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define SOL_XDP 283
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #ifndef AF_XDP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define AF_XDP 44
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #ifndef PF_XDP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define PF_XDP AF_XDP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct xsk_umem {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct xsk_ring_prod *fill_save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct xsk_ring_cons *comp_save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) char *umem_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct xsk_umem_config config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) int fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) int refcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct list_head ctx_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) bool rx_ring_setup_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) bool tx_ring_setup_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct xsk_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct xsk_ring_prod *fill;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct xsk_ring_cons *comp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) __u32 queue_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct xsk_umem *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) int refcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) int ifindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) int prog_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) int xsks_map_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) char ifname[IFNAMSIZ];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct xsk_socket {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct xsk_ring_cons *rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct xsk_ring_prod *tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) __u64 outstanding_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) struct xsk_ctx *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct xsk_socket_config config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) int fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct xsk_nl_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) bool xdp_prog_attached;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int ifindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) int fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) /* Up until and including Linux 5.3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct xdp_ring_offset_v1 {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) __u64 producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) __u64 consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) __u64 desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) /* Up until and including Linux 5.3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct xdp_mmap_offsets_v1 {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct xdp_ring_offset_v1 rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) struct xdp_ring_offset_v1 tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct xdp_ring_offset_v1 fr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct xdp_ring_offset_v1 cr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) int xsk_umem__fd(const struct xsk_umem *umem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) return umem ? umem->fd : -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) int xsk_socket__fd(const struct xsk_socket *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return xsk ? xsk->fd : -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static bool xsk_page_aligned(void *buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) unsigned long addr = (unsigned long)buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return !(addr & (getpagesize() - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static void xsk_set_umem_config(struct xsk_umem_config *cfg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) const struct xsk_umem_config *usr_cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (!usr_cfg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) cfg->fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) cfg->comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) cfg->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) cfg->frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) cfg->flags = XSK_UMEM__DEFAULT_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) cfg->fill_size = usr_cfg->fill_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) cfg->comp_size = usr_cfg->comp_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) cfg->frame_size = usr_cfg->frame_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) cfg->frame_headroom = usr_cfg->frame_headroom;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) cfg->flags = usr_cfg->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static int xsk_set_xdp_socket_config(struct xsk_socket_config *cfg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) const struct xsk_socket_config *usr_cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) if (!usr_cfg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) cfg->rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) cfg->tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) cfg->libbpf_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) cfg->xdp_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) cfg->bind_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (usr_cfg->libbpf_flags & ~XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) cfg->rx_size = usr_cfg->rx_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) cfg->tx_size = usr_cfg->tx_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) cfg->libbpf_flags = usr_cfg->libbpf_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) cfg->xdp_flags = usr_cfg->xdp_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) cfg->bind_flags = usr_cfg->bind_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static void xsk_mmap_offsets_v1(struct xdp_mmap_offsets *off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct xdp_mmap_offsets_v1 off_v1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /* getsockopt on a kernel <= 5.3 has no flags fields.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * Copy over the offsets to the correct places in the >=5.4 format
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * and put the flags where they would have been on that kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) memcpy(&off_v1, off, sizeof(off_v1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) off->rx.producer = off_v1.rx.producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) off->rx.consumer = off_v1.rx.consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) off->rx.desc = off_v1.rx.desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) off->rx.flags = off_v1.rx.consumer + sizeof(__u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) off->tx.producer = off_v1.tx.producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) off->tx.consumer = off_v1.tx.consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) off->tx.desc = off_v1.tx.desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) off->tx.flags = off_v1.tx.consumer + sizeof(__u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) off->fr.producer = off_v1.fr.producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) off->fr.consumer = off_v1.fr.consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) off->fr.desc = off_v1.fr.desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) off->fr.flags = off_v1.fr.consumer + sizeof(__u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) off->cr.producer = off_v1.cr.producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) off->cr.consumer = off_v1.cr.consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) off->cr.desc = off_v1.cr.desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) off->cr.flags = off_v1.cr.consumer + sizeof(__u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) static int xsk_get_mmap_offsets(int fd, struct xdp_mmap_offsets *off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) socklen_t optlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) optlen = sizeof(*off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) err = getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, off, &optlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) if (optlen == sizeof(*off))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (optlen == sizeof(struct xdp_mmap_offsets_v1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) xsk_mmap_offsets_v1(off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static int xsk_create_umem_rings(struct xsk_umem *umem, int fd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct xsk_ring_prod *fill,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct xsk_ring_cons *comp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) struct xdp_mmap_offsets off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) void *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) err = setsockopt(fd, SOL_XDP, XDP_UMEM_FILL_RING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) &umem->config.fill_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) sizeof(umem->config.fill_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) err = setsockopt(fd, SOL_XDP, XDP_UMEM_COMPLETION_RING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) &umem->config.comp_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) sizeof(umem->config.comp_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) return -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) err = xsk_get_mmap_offsets(fd, &off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) return -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) map = mmap(NULL, off.fr.desc + umem->config.fill_size * sizeof(__u64),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) XDP_UMEM_PGOFF_FILL_RING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (map == MAP_FAILED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) return -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) fill->mask = umem->config.fill_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) fill->size = umem->config.fill_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) fill->producer = map + off.fr.producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) fill->consumer = map + off.fr.consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) fill->flags = map + off.fr.flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) fill->ring = map + off.fr.desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) fill->cached_cons = umem->config.fill_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) map = mmap(NULL, off.cr.desc + umem->config.comp_size * sizeof(__u64),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) XDP_UMEM_PGOFF_COMPLETION_RING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (map == MAP_FAILED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) goto out_mmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) comp->mask = umem->config.comp_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) comp->size = umem->config.comp_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) comp->producer = map + off.cr.producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) comp->consumer = map + off.cr.consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) comp->flags = map + off.cr.flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) comp->ring = map + off.cr.desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) out_mmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) munmap(map, off.fr.desc + umem->config.fill_size * sizeof(__u64));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) __u64 size, struct xsk_ring_prod *fill,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct xsk_ring_cons *comp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) const struct xsk_umem_config *usr_config)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) struct xdp_umem_reg mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) struct xsk_umem *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (!umem_area || !umem_ptr || !fill || !comp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) if (!size && !xsk_page_aligned(umem_area))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) umem = calloc(1, sizeof(*umem));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) if (!umem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) umem->fd = socket(AF_XDP, SOCK_RAW, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) if (umem->fd < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) goto out_umem_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) umem->umem_area = umem_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) INIT_LIST_HEAD(&umem->ctx_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) xsk_set_umem_config(&umem->config, usr_config);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) memset(&mr, 0, sizeof(mr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) mr.addr = (uintptr_t)umem_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) mr.len = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) mr.chunk_size = umem->config.frame_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) mr.headroom = umem->config.frame_headroom;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) mr.flags = umem->config.flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) err = setsockopt(umem->fd, SOL_XDP, XDP_UMEM_REG, &mr, sizeof(mr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) goto out_socket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) err = xsk_create_umem_rings(umem, umem->fd, fill, comp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) goto out_socket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) umem->fill_save = fill;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) umem->comp_save = comp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) *umem_ptr = umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) out_socket:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) close(umem->fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) out_umem_alloc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) free(umem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) struct xsk_umem_config_v1 {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) __u32 fill_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) __u32 comp_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) __u32 frame_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) __u32 frame_headroom;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) __u64 size, struct xsk_ring_prod *fill,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) struct xsk_ring_cons *comp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) const struct xsk_umem_config *usr_config)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct xsk_umem_config config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) memcpy(&config, usr_config, sizeof(struct xsk_umem_config_v1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) config.flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return xsk_umem__create_v0_0_4(umem_ptr, umem_area, size, fill, comp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) &config);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static int xsk_load_xdp_prog(struct xsk_socket *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static const int log_buf_size = 16 * 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) struct xsk_ctx *ctx = xsk->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) char log_buf[log_buf_size];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) int err, prog_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) /* This is the C-program:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * int ret, index = ctx->rx_queue_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * // A set entry here means that the correspnding queue_id
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * // has an active AF_XDP socket bound to it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * ret = bpf_redirect_map(&xsks_map, index, XDP_PASS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * // Fallback for pre-5.3 kernels, not supporting default
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * // action in the flags parameter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * if (bpf_map_lookup_elem(&xsks_map, &index))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * return bpf_redirect_map(&xsks_map, index, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * return XDP_PASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct bpf_insn prog[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) /* r2 = *(u32 *)(r1 + 16) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /* *(u32 *)(r10 - 4) = r2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -4),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) /* r1 = xskmap[] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /* r3 = XDP_PASS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) BPF_MOV64_IMM(BPF_REG_3, 2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) /* call bpf_redirect_map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) BPF_EMIT_CALL(BPF_FUNC_redirect_map),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) /* if w0 != 0 goto pc+13 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) BPF_JMP32_IMM(BPF_JSGT, BPF_REG_0, 0, 13),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) /* r2 = r10 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) /* r2 += -4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /* r1 = xskmap[] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) /* call bpf_map_lookup_elem */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) /* r1 = r0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /* r0 = XDP_PASS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) BPF_MOV64_IMM(BPF_REG_0, 2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) /* if r1 == 0 goto pc+5 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /* r2 = *(u32 *)(r10 - 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_10, -4),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) /* r1 = xskmap[] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) /* r3 = 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) BPF_MOV64_IMM(BPF_REG_3, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) /* call bpf_redirect_map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) BPF_EMIT_CALL(BPF_FUNC_redirect_map),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /* The jumps are to this instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) BPF_EXIT_INSN(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) prog_fd = bpf_load_program(BPF_PROG_TYPE_XDP, prog, insns_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) "LGPL-2.1 or BSD-2-Clause", 0, log_buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) log_buf_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (prog_fd < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) pr_warn("BPF log buffer:\n%s", log_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) return prog_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) err = bpf_set_link_xdp_fd(xsk->ctx->ifindex, prog_fd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) xsk->config.xdp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) close(prog_fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) ctx->prog_fd = prog_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) static int xsk_get_max_queues(struct xsk_socket *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct ethtool_channels channels = { .cmd = ETHTOOL_GCHANNELS };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) struct xsk_ctx *ctx = xsk->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) struct ifreq ifr = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) int fd, err, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) fd = socket(AF_INET, SOCK_DGRAM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) if (fd < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) return -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) ifr.ifr_data = (void *)&channels;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) memcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) ifr.ifr_name[IFNAMSIZ - 1] = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) err = ioctl(fd, SIOCETHTOOL, &ifr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) if (err && errno != EOPNOTSUPP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) ret = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) /* If the device says it has no channels, then all traffic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * is sent to a single stream, so max queues = 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) /* Take the max of rx, tx, combined. Drivers return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * the number of channels in different ways.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) ret = max(channels.max_rx, channels.max_tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) ret = max(ret, (int)channels.max_combined);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) close(fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) static int xsk_create_bpf_maps(struct xsk_socket *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) struct xsk_ctx *ctx = xsk->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) int max_queues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) int fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) max_queues = xsk_get_max_queues(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (max_queues < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return max_queues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) fd = bpf_create_map_name(BPF_MAP_TYPE_XSKMAP, "xsks_map",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) sizeof(int), sizeof(int), max_queues, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (fd < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) ctx->xsks_map_fd = fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static void xsk_delete_bpf_maps(struct xsk_socket *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) struct xsk_ctx *ctx = xsk->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) bpf_map_delete_elem(ctx->xsks_map_fd, &ctx->queue_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) close(ctx->xsks_map_fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) static int xsk_lookup_bpf_maps(struct xsk_socket *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) __u32 i, *map_ids, num_maps, prog_len = sizeof(struct bpf_prog_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) __u32 map_len = sizeof(struct bpf_map_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) struct bpf_prog_info prog_info = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) struct xsk_ctx *ctx = xsk->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) struct bpf_map_info map_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) int fd, err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) num_maps = prog_info.nr_map_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) map_ids = calloc(prog_info.nr_map_ids, sizeof(*map_ids));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (!map_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) memset(&prog_info, 0, prog_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) prog_info.nr_map_ids = num_maps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) prog_info.map_ids = (__u64)(unsigned long)map_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) goto out_map_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) ctx->xsks_map_fd = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) for (i = 0; i < prog_info.nr_map_ids; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) fd = bpf_map_get_fd_by_id(map_ids[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (fd < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) memset(&map_info, 0, map_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) err = bpf_obj_get_info_by_fd(fd, &map_info, &map_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) close(fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (!strncmp(map_info.name, "xsks_map", sizeof(map_info.name))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) ctx->xsks_map_fd = fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) close(fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) if (ctx->xsks_map_fd == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) err = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) out_map_ids:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) free(map_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) static int xsk_set_bpf_maps(struct xsk_socket *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) struct xsk_ctx *ctx = xsk->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) return bpf_map_update_elem(ctx->xsks_map_fd, &ctx->queue_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) &xsk->fd, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) struct xsk_ctx *ctx = xsk->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) __u32 prog_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) xsk->config.xdp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (!prog_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) err = xsk_create_bpf_maps(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) err = xsk_load_xdp_prog(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) xsk_delete_bpf_maps(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) if (ctx->prog_fd < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) return -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) err = xsk_lookup_bpf_maps(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) close(ctx->prog_fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) if (xsk->rx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) err = xsk_set_bpf_maps(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) xsk_delete_bpf_maps(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) close(ctx->prog_fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) static struct xsk_ctx *xsk_get_ctx(struct xsk_umem *umem, int ifindex,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) __u32 queue_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) struct xsk_ctx *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) if (list_empty(&umem->ctx_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) list_for_each_entry(ctx, &umem->ctx_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (ctx->ifindex == ifindex && ctx->queue_id == queue_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) ctx->refcount++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) return ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) static void xsk_put_ctx(struct xsk_ctx *ctx, bool unmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) struct xsk_umem *umem = ctx->umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) struct xdp_mmap_offsets off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (--ctx->refcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (!unmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) err = xsk_get_mmap_offsets(umem->fd, &off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) munmap(ctx->fill->ring - off.fr.desc, off.fr.desc + umem->config.fill_size *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) sizeof(__u64));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) munmap(ctx->comp->ring - off.cr.desc, off.cr.desc + umem->config.comp_size *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) sizeof(__u64));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) list_del(&ctx->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) free(ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) struct xsk_umem *umem, int ifindex,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) const char *ifname, __u32 queue_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) struct xsk_ring_prod *fill,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) struct xsk_ring_cons *comp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) struct xsk_ctx *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) ctx = calloc(1, sizeof(*ctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) if (!ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) if (!umem->fill_save) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) err = xsk_create_umem_rings(umem, xsk->fd, fill, comp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) free(ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) } else if (umem->fill_save != fill || umem->comp_save != comp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) /* Copy over rings to new structs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) memcpy(fill, umem->fill_save, sizeof(*fill));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) memcpy(comp, umem->comp_save, sizeof(*comp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) ctx->ifindex = ifindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) ctx->refcount = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) ctx->umem = umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) ctx->queue_id = queue_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) memcpy(ctx->ifname, ifname, IFNAMSIZ - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) ctx->ifname[IFNAMSIZ - 1] = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) ctx->fill = fill;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) ctx->comp = comp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) list_add(&ctx->list, &umem->ctx_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) return ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) const char *ifname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) __u32 queue_id, struct xsk_umem *umem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) struct xsk_ring_cons *rx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) struct xsk_ring_prod *tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) struct xsk_ring_prod *fill,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) struct xsk_ring_cons *comp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) const struct xsk_socket_config *usr_config)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) bool unmap, rx_setup_done = false, tx_setup_done = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) void *rx_map = NULL, *tx_map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) struct sockaddr_xdp sxdp = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) struct xdp_mmap_offsets off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) struct xsk_socket *xsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) struct xsk_ctx *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) int err, ifindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) if (!umem || !xsk_ptr || !(rx || tx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) unmap = umem->fill_save != fill;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) xsk = calloc(1, sizeof(*xsk));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (!xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) err = xsk_set_xdp_socket_config(&xsk->config, usr_config);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) goto out_xsk_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) xsk->outstanding_tx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) ifindex = if_nametoindex(ifname);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (!ifindex) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) goto out_xsk_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) if (umem->refcount++ > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) xsk->fd = socket(AF_XDP, SOCK_RAW, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (xsk->fd < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) goto out_xsk_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) xsk->fd = umem->fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) rx_setup_done = umem->rx_ring_setup_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) tx_setup_done = umem->tx_ring_setup_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) ctx = xsk_get_ctx(umem, ifindex, queue_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (!ctx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) if (!fill || !comp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) err = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) goto out_socket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) ctx = xsk_create_ctx(xsk, umem, ifindex, ifname, queue_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) fill, comp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (!ctx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) goto out_socket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) xsk->ctx = ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (rx && !rx_setup_done) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) &xsk->config.rx_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) sizeof(xsk->config.rx_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) goto out_put_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) if (xsk->fd == umem->fd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) umem->rx_ring_setup_done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (tx && !tx_setup_done) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) err = setsockopt(xsk->fd, SOL_XDP, XDP_TX_RING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) &xsk->config.tx_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) sizeof(xsk->config.tx_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) goto out_put_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) if (xsk->fd == umem->fd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) umem->tx_ring_setup_done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) err = xsk_get_mmap_offsets(xsk->fd, &off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) goto out_put_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (rx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) rx_map = mmap(NULL, off.rx.desc +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) xsk->config.rx_size * sizeof(struct xdp_desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) xsk->fd, XDP_PGOFF_RX_RING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) if (rx_map == MAP_FAILED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) goto out_put_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) rx->mask = xsk->config.rx_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) rx->size = xsk->config.rx_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) rx->producer = rx_map + off.rx.producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) rx->consumer = rx_map + off.rx.consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) rx->flags = rx_map + off.rx.flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) rx->ring = rx_map + off.rx.desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) rx->cached_prod = *rx->producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) rx->cached_cons = *rx->consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) xsk->rx = rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) if (tx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) tx_map = mmap(NULL, off.tx.desc +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) xsk->config.tx_size * sizeof(struct xdp_desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) xsk->fd, XDP_PGOFF_TX_RING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) if (tx_map == MAP_FAILED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) goto out_mmap_rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) tx->mask = xsk->config.tx_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) tx->size = xsk->config.tx_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) tx->producer = tx_map + off.tx.producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) tx->consumer = tx_map + off.tx.consumer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) tx->flags = tx_map + off.tx.flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) tx->ring = tx_map + off.tx.desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) tx->cached_prod = *tx->producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) /* cached_cons is r->size bigger than the real consumer pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) * See xsk_prod_nb_free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) tx->cached_cons = *tx->consumer + xsk->config.tx_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) xsk->tx = tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) sxdp.sxdp_family = PF_XDP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) sxdp.sxdp_ifindex = ctx->ifindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) sxdp.sxdp_queue_id = ctx->queue_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) if (umem->refcount > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) sxdp.sxdp_flags |= XDP_SHARED_UMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) sxdp.sxdp_shared_umem_fd = umem->fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) sxdp.sxdp_flags = xsk->config.bind_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) err = bind(xsk->fd, (struct sockaddr *)&sxdp, sizeof(sxdp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) err = -errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) goto out_mmap_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) ctx->prog_fd = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) err = xsk_setup_xdp_prog(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) goto out_mmap_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) *xsk_ptr = xsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) umem->fill_save = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) umem->comp_save = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) out_mmap_tx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) munmap(tx_map, off.tx.desc +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) xsk->config.tx_size * sizeof(struct xdp_desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) out_mmap_rx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (rx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) munmap(rx_map, off.rx.desc +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) xsk->config.rx_size * sizeof(struct xdp_desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) out_put_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) xsk_put_ctx(ctx, unmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) out_socket:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) if (--umem->refcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) close(xsk->fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) out_xsk_alloc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) free(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) __u32 queue_id, struct xsk_umem *umem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) struct xsk_ring_cons *rx, struct xsk_ring_prod *tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) const struct xsk_socket_config *usr_config)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) if (!umem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) return xsk_socket__create_shared(xsk_ptr, ifname, queue_id, umem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) rx, tx, umem->fill_save,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) umem->comp_save, usr_config);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) int xsk_umem__delete(struct xsk_umem *umem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) struct xdp_mmap_offsets off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) if (!umem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) if (umem->refcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) err = xsk_get_mmap_offsets(umem->fd, &off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (!err && umem->fill_save && umem->comp_save) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) munmap(umem->fill_save->ring - off.fr.desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) off.fr.desc + umem->config.fill_size * sizeof(__u64));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) munmap(umem->comp_save->ring - off.cr.desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) off.cr.desc + umem->config.comp_size * sizeof(__u64));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) close(umem->fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) free(umem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) void xsk_socket__delete(struct xsk_socket *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) size_t desc_sz = sizeof(struct xdp_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) struct xdp_mmap_offsets off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) struct xsk_umem *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) struct xsk_ctx *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) if (!xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) ctx = xsk->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) umem = ctx->umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) if (ctx->prog_fd != -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) xsk_delete_bpf_maps(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) close(ctx->prog_fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) err = xsk_get_mmap_offsets(xsk->fd, &off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) if (xsk->rx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) munmap(xsk->rx->ring - off.rx.desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) off.rx.desc + xsk->config.rx_size * desc_sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) if (xsk->tx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) munmap(xsk->tx->ring - off.tx.desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) off.tx.desc + xsk->config.tx_size * desc_sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) xsk_put_ctx(ctx, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) umem->refcount--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) /* Do not close an fd that also has an associated umem connected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) * to it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) if (xsk->fd != umem->fd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) close(xsk->fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) free(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate