^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Copyright(c) 2017 - 2018 Intel Corporation. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <asm/barrier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <getopt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <libgen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/if_link.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/if_xdp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/if_ether.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/ip.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/limits.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/udp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <arpa/inet.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <locale.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <net/ethernet.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <net/if.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <poll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <pthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <stdbool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <stdio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <stdlib.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <sys/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <sys/resource.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <sys/socket.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <sys/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <time.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <unistd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <bpf/libbpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <bpf/xsk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <bpf/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include "xdpsock.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #ifndef SOL_XDP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define SOL_XDP 283
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #ifndef AF_XDP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define AF_XDP 44
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #ifndef PF_XDP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define PF_XDP AF_XDP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define NUM_FRAMES (4 * 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define MIN_PKT_SIZE 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define DEBUG_HEXDUMP 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) typedef __u64 u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) typedef __u32 u32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) typedef __u16 u16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) typedef __u8 u8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static unsigned long prev_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) enum benchmark_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) BENCH_RXDROP = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) BENCH_TXONLY = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) BENCH_L2FWD = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) static enum benchmark_type opt_bench = BENCH_RXDROP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static const char *opt_if = "";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static int opt_ifindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static int opt_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) static unsigned long opt_duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static unsigned long start_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static bool benchmark_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static u32 opt_batch_size = 64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static int opt_pkt_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static u16 opt_pkt_size = MIN_PKT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static u32 opt_pkt_fill_pattern = 0x12345678;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static bool opt_extra_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static bool opt_quiet;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static bool opt_app_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static const char *opt_irq_str = "";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static u32 irq_no;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) static int irqs_at_init = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static int opt_poll;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static int opt_interval = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static u32 opt_umem_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static int opt_unaligned_chunks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static int opt_mmap_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static int opt_timeout = 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static bool opt_need_wakeup = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static u32 opt_num_xsks = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) static u32 prog_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) struct xsk_ring_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) unsigned long rx_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) unsigned long tx_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) unsigned long rx_dropped_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) unsigned long rx_invalid_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) unsigned long tx_invalid_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) unsigned long rx_full_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) unsigned long rx_fill_empty_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) unsigned long tx_empty_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) unsigned long prev_rx_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) unsigned long prev_tx_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) unsigned long prev_rx_dropped_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) unsigned long prev_rx_invalid_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) unsigned long prev_tx_invalid_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) unsigned long prev_rx_full_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) unsigned long prev_rx_fill_empty_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) unsigned long prev_tx_empty_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct xsk_driver_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) unsigned long intrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) unsigned long prev_intrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct xsk_app_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) unsigned long rx_empty_polls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) unsigned long fill_fail_polls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) unsigned long copy_tx_sendtos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) unsigned long tx_wakeup_sendtos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) unsigned long opt_polls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) unsigned long prev_rx_empty_polls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) unsigned long prev_fill_fail_polls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) unsigned long prev_copy_tx_sendtos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) unsigned long prev_tx_wakeup_sendtos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) unsigned long prev_opt_polls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct xsk_umem_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct xsk_ring_prod fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) struct xsk_ring_cons cq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct xsk_umem *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) void *buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct xsk_socket_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) struct xsk_ring_cons rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct xsk_ring_prod tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct xsk_umem_info *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct xsk_socket *xsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct xsk_ring_stats ring_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) struct xsk_app_stats app_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) struct xsk_driver_stats drv_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) u32 outstanding_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static int num_socks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) struct xsk_socket_info *xsks[MAX_SOCKS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static unsigned long get_nsecs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) struct timespec ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) clock_gettime(CLOCK_MONOTONIC, &ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) return ts.tv_sec * 1000000000UL + ts.tv_nsec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static void print_benchmark(bool running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) const char *bench_str = "INVALID";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (opt_bench == BENCH_RXDROP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) bench_str = "rxdrop";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) else if (opt_bench == BENCH_TXONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) bench_str = "txonly";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) else if (opt_bench == BENCH_L2FWD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) bench_str = "l2fwd";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) printf("%s:%d %s ", opt_if, opt_queue, bench_str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (opt_xdp_flags & XDP_FLAGS_SKB_MODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) printf("xdp-skb ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) printf("xdp-drv ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) printf(" ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (opt_poll)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) printf("poll() ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (running) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) printf("running...");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) fflush(stdout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) static int xsk_get_xdp_stats(int fd, struct xsk_socket_info *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) struct xdp_statistics stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) socklen_t optlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) optlen = sizeof(stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (optlen == sizeof(struct xdp_statistics)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) xsk->ring_stats.rx_dropped_npkts = stats.rx_dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) xsk->ring_stats.rx_invalid_npkts = stats.rx_invalid_descs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) xsk->ring_stats.tx_invalid_npkts = stats.tx_invalid_descs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) xsk->ring_stats.rx_full_npkts = stats.rx_ring_full;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) xsk->ring_stats.rx_fill_empty_npkts = stats.rx_fill_ring_empty_descs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) xsk->ring_stats.tx_empty_npkts = stats.tx_ring_empty_descs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static void dump_app_stats(long dt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) for (i = 0; i < num_socks && xsks[i]; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) char *fmt = "%-18s %'-14.0f %'-14lu\n";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) double rx_empty_polls_ps, fill_fail_polls_ps, copy_tx_sendtos_ps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) tx_wakeup_sendtos_ps, opt_polls_ps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) rx_empty_polls_ps = (xsks[i]->app_stats.rx_empty_polls -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) xsks[i]->app_stats.prev_rx_empty_polls) * 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) fill_fail_polls_ps = (xsks[i]->app_stats.fill_fail_polls -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) xsks[i]->app_stats.prev_fill_fail_polls) * 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) copy_tx_sendtos_ps = (xsks[i]->app_stats.copy_tx_sendtos -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) xsks[i]->app_stats.prev_copy_tx_sendtos) * 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) tx_wakeup_sendtos_ps = (xsks[i]->app_stats.tx_wakeup_sendtos -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) xsks[i]->app_stats.prev_tx_wakeup_sendtos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) opt_polls_ps = (xsks[i]->app_stats.opt_polls -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) xsks[i]->app_stats.prev_opt_polls) * 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) printf("\n%-18s %-14s %-14s\n", "", "calls/s", "count");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) printf(fmt, "rx empty polls", rx_empty_polls_ps, xsks[i]->app_stats.rx_empty_polls);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) printf(fmt, "fill fail polls", fill_fail_polls_ps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) xsks[i]->app_stats.fill_fail_polls);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) printf(fmt, "copy tx sendtos", copy_tx_sendtos_ps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) xsks[i]->app_stats.copy_tx_sendtos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) printf(fmt, "tx wakeup sendtos", tx_wakeup_sendtos_ps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) xsks[i]->app_stats.tx_wakeup_sendtos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) printf(fmt, "opt polls", opt_polls_ps, xsks[i]->app_stats.opt_polls);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) xsks[i]->app_stats.prev_rx_empty_polls = xsks[i]->app_stats.rx_empty_polls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) xsks[i]->app_stats.prev_fill_fail_polls = xsks[i]->app_stats.fill_fail_polls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) xsks[i]->app_stats.prev_copy_tx_sendtos = xsks[i]->app_stats.copy_tx_sendtos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) xsks[i]->app_stats.prev_tx_wakeup_sendtos = xsks[i]->app_stats.tx_wakeup_sendtos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) xsks[i]->app_stats.prev_opt_polls = xsks[i]->app_stats.opt_polls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) static bool get_interrupt_number(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) FILE *f_int_proc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) char line[4096];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) bool found = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) f_int_proc = fopen("/proc/interrupts", "r");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) if (f_int_proc == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) printf("Failed to open /proc/interrupts.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) while (!feof(f_int_proc) && !found) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /* Make sure to read a full line at a time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (fgets(line, sizeof(line), f_int_proc) == NULL ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) line[strlen(line) - 1] != '\n') {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) printf("Error reading from interrupts file\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /* Extract interrupt number from line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (strstr(line, opt_irq_str) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) irq_no = atoi(line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) found = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) fclose(f_int_proc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static int get_irqs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) char count_path[PATH_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) int total_intrs = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) FILE *f_count_proc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) char line[4096];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) snprintf(count_path, sizeof(count_path),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) "/sys/kernel/irq/%i/per_cpu_count", irq_no);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) f_count_proc = fopen(count_path, "r");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) if (f_count_proc == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) printf("Failed to open %s\n", count_path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) return total_intrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (fgets(line, sizeof(line), f_count_proc) == NULL ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) line[strlen(line) - 1] != '\n') {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) printf("Error reading from %s\n", count_path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) static const char com[2] = ",";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) char *token;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) total_intrs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) token = strtok(line, com);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) while (token != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /* sum up interrupts across all cores */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) total_intrs += atoi(token);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) token = strtok(NULL, com);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) fclose(f_count_proc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) return total_intrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) static void dump_driver_stats(long dt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) for (i = 0; i < num_socks && xsks[i]; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) char *fmt = "%-18s %'-14.0f %'-14lu\n";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) double intrs_ps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) int n_ints = get_irqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) if (n_ints < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) printf("error getting intr info for intr %i\n", irq_no);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) xsks[i]->drv_stats.intrs = n_ints - irqs_at_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) intrs_ps = (xsks[i]->drv_stats.intrs - xsks[i]->drv_stats.prev_intrs) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) printf("\n%-18s %-14s %-14s\n", "", "intrs/s", "count");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) printf(fmt, "irqs", intrs_ps, xsks[i]->drv_stats.intrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) xsks[i]->drv_stats.prev_intrs = xsks[i]->drv_stats.intrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) static void dump_stats(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) unsigned long now = get_nsecs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) long dt = now - prev_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) prev_time = now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) for (i = 0; i < num_socks && xsks[i]; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) char *fmt = "%-18s %'-14.0f %'-14lu\n";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) double rx_pps, tx_pps, dropped_pps, rx_invalid_pps, full_pps, fill_empty_pps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) tx_invalid_pps, tx_empty_pps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) rx_pps = (xsks[i]->ring_stats.rx_npkts - xsks[i]->ring_stats.prev_rx_npkts) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) tx_pps = (xsks[i]->ring_stats.tx_npkts - xsks[i]->ring_stats.prev_tx_npkts) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) printf("\n sock%d@", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) print_benchmark(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) printf("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) printf("%-18s %-14s %-14s %-14.2f\n", "", "pps", "pkts",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) dt / 1000000000.);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) printf(fmt, "rx", rx_pps, xsks[i]->ring_stats.rx_npkts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) printf(fmt, "tx", tx_pps, xsks[i]->ring_stats.tx_npkts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) xsks[i]->ring_stats.prev_rx_npkts = xsks[i]->ring_stats.rx_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) xsks[i]->ring_stats.prev_tx_npkts = xsks[i]->ring_stats.tx_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (opt_extra_stats) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (!xsk_get_xdp_stats(xsk_socket__fd(xsks[i]->xsk), xsks[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) dropped_pps = (xsks[i]->ring_stats.rx_dropped_npkts -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) xsks[i]->ring_stats.prev_rx_dropped_npkts) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) rx_invalid_pps = (xsks[i]->ring_stats.rx_invalid_npkts -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) xsks[i]->ring_stats.prev_rx_invalid_npkts) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) tx_invalid_pps = (xsks[i]->ring_stats.tx_invalid_npkts -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) xsks[i]->ring_stats.prev_tx_invalid_npkts) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) full_pps = (xsks[i]->ring_stats.rx_full_npkts -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) xsks[i]->ring_stats.prev_rx_full_npkts) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) fill_empty_pps = (xsks[i]->ring_stats.rx_fill_empty_npkts -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) xsks[i]->ring_stats.prev_rx_fill_empty_npkts) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) tx_empty_pps = (xsks[i]->ring_stats.tx_empty_npkts -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) xsks[i]->ring_stats.prev_tx_empty_npkts) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) 1000000000. / dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) printf(fmt, "rx dropped", dropped_pps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) xsks[i]->ring_stats.rx_dropped_npkts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) printf(fmt, "rx invalid", rx_invalid_pps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) xsks[i]->ring_stats.rx_invalid_npkts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) printf(fmt, "tx invalid", tx_invalid_pps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) xsks[i]->ring_stats.tx_invalid_npkts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) printf(fmt, "rx queue full", full_pps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) xsks[i]->ring_stats.rx_full_npkts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) printf(fmt, "fill ring empty", fill_empty_pps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) xsks[i]->ring_stats.rx_fill_empty_npkts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) printf(fmt, "tx ring empty", tx_empty_pps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) xsks[i]->ring_stats.tx_empty_npkts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) xsks[i]->ring_stats.prev_rx_dropped_npkts =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) xsks[i]->ring_stats.rx_dropped_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) xsks[i]->ring_stats.prev_rx_invalid_npkts =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) xsks[i]->ring_stats.rx_invalid_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) xsks[i]->ring_stats.prev_tx_invalid_npkts =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) xsks[i]->ring_stats.tx_invalid_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) xsks[i]->ring_stats.prev_rx_full_npkts =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) xsks[i]->ring_stats.rx_full_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) xsks[i]->ring_stats.prev_rx_fill_empty_npkts =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) xsks[i]->ring_stats.rx_fill_empty_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) xsks[i]->ring_stats.prev_tx_empty_npkts =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) xsks[i]->ring_stats.tx_empty_npkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) printf("%-15s\n", "Error retrieving extra stats");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if (opt_app_stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) dump_app_stats(dt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (irq_no)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) dump_driver_stats(dt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) static bool is_benchmark_done(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (opt_duration > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) unsigned long dt = (get_nsecs() - start_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (dt >= opt_duration)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) benchmark_done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) return benchmark_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) static void *poller(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) (void)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) while (!is_benchmark_done()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) sleep(opt_interval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) dump_stats();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) static void remove_xdp_program(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) u32 curr_prog_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) printf("bpf_get_link_xdp_id failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (prog_id == curr_prog_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) else if (!curr_prog_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) printf("couldn't find a prog id on a given interface\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) printf("program on interface changed, not removing\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) static void int_exit(int sig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) benchmark_done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) static void xdpsock_cleanup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) struct xsk_umem *umem = xsks[0]->umem->umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) dump_stats();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) for (i = 0; i < num_socks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) xsk_socket__delete(xsks[i]->xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) (void)xsk_umem__delete(umem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) remove_xdp_program();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) static void __exit_with_error(int error, const char *file, const char *func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) int line)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) line, error, strerror(error));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) dump_stats();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) remove_xdp_program();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) __LINE__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) static void swap_mac_addresses(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) struct ether_header *eth = (struct ether_header *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) struct ether_addr tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) tmp = *src_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) *src_addr = *dst_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) *dst_addr = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) static void hex_dump(void *pkt, size_t length, u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) const unsigned char *address = (unsigned char *)pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) const unsigned char *line = address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) size_t line_size = 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) unsigned char c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) char buf[32];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (!DEBUG_HEXDUMP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) sprintf(buf, "addr=%llu", addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) printf("length = %zu\n", length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) printf("%s | ", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) while (length-- > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) printf("%02X ", *address++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (!(++i % line_size) || (length == 0 && i % line_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (length == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) while (i++ % line_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) printf("__ ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) printf(" | "); /* right close */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) while (line < address) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) c = *line++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) printf("%c", (c < 33 || c == 255) ? 0x2E : c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) printf("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (length > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) printf("%s | ", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) printf("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) static void *memset32_htonl(void *dest, u32 val, u32 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) u32 *ptr = (u32 *)dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) val = htonl(val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) for (i = 0; i < (size & (~0x3)); i += 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) ptr[i >> 2] = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) for (; i < size; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) ((char *)dest)[i] = ((char *)&val)[i & 3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) return dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * This function code has been taken from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * Linux kernel lib/checksum.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) static inline unsigned short from32to16(unsigned int x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) /* add up 16-bit and 16-bit for 16+c bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) x = (x & 0xffff) + (x >> 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) /* add up carry.. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) x = (x & 0xffff) + (x >> 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) return x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * This function code has been taken from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * Linux kernel lib/checksum.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) static unsigned int do_csum(const unsigned char *buff, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) unsigned int result = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) int odd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (len <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) odd = 1 & (unsigned long)buff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) if (odd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) #ifdef __LITTLE_ENDIAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) result += (*buff << 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) result = *buff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) len--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) buff++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (len >= 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (2 & (unsigned long)buff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) result += *(unsigned short *)buff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) len -= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) buff += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (len >= 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) const unsigned char *end = buff +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) ((unsigned int)len & ~3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) unsigned int carry = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) unsigned int w = *(unsigned int *)buff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) buff += 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) result += carry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) result += w;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) carry = (w > result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) } while (buff < end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) result += carry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) result = (result & 0xffff) + (result >> 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (len & 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) result += *(unsigned short *)buff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) buff += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) if (len & 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) #ifdef __LITTLE_ENDIAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) result += *buff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) result += (*buff << 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) result = from32to16(result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) if (odd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) result = ((result >> 8) & 0xff) | ((result & 0xff) << 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) __sum16 ip_fast_csum(const void *iph, unsigned int ihl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * This is a version of ip_compute_csum() optimized for IP headers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * which always checksum on 4 octet boundaries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) * This function code has been taken from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * Linux kernel lib/checksum.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) return (__force __sum16)~do_csum(iph, ihl * 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * Fold a partial checksum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * This function code has been taken from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) * Linux kernel include/asm-generic/checksum.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) static inline __sum16 csum_fold(__wsum csum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) u32 sum = (__force u32)csum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) sum = (sum & 0xffff) + (sum >> 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) sum = (sum & 0xffff) + (sum >> 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) return (__force __sum16)~sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * This function code has been taken from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * Linux kernel lib/checksum.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) static inline u32 from64to32(u64 x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) /* add up 32-bit and 32-bit for 32+c bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) x = (x & 0xffffffff) + (x >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) /* add up carry.. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) x = (x & 0xffffffff) + (x >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) return (u32)x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) __u32 len, __u8 proto, __wsum sum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) * This function code has been taken from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) * Linux kernel lib/checksum.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) __u32 len, __u8 proto, __wsum sum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) unsigned long long s = (__force u32)sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) s += (__force u32)saddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) s += (__force u32)daddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) #ifdef __BIG_ENDIAN__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) s += proto + len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) s += (proto + len) << 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) return (__force __wsum)from64to32(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * This function has been taken from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * Linux kernel include/asm-generic/checksum.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) static inline __sum16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) __u8 proto, __wsum sum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) static inline u16 udp_csum(u32 saddr, u32 daddr, u32 len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) u8 proto, u16 *udp_pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) u32 csum = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) u32 cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) /* udp hdr and data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) for (; cnt < len; cnt += 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) csum += udp_pkt[cnt >> 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) return csum_tcpudp_magic(saddr, daddr, len, proto, csum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) #define ETH_FCS_SIZE 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) #define PKT_HDR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr) + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) sizeof(struct udphdr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) #define PKT_SIZE (opt_pkt_size - ETH_FCS_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) #define IP_PKT_SIZE (PKT_SIZE - sizeof(struct ethhdr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) #define UDP_PKT_SIZE (IP_PKT_SIZE - sizeof(struct iphdr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) #define UDP_PKT_DATA_SIZE (UDP_PKT_SIZE - sizeof(struct udphdr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) static u8 pkt_data[XSK_UMEM__DEFAULT_FRAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) static void gen_eth_hdr_data(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) struct udphdr *udp_hdr = (struct udphdr *)(pkt_data +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) sizeof(struct ethhdr) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) sizeof(struct iphdr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) struct iphdr *ip_hdr = (struct iphdr *)(pkt_data +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) sizeof(struct ethhdr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) struct ethhdr *eth_hdr = (struct ethhdr *)pkt_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) /* ethernet header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) memcpy(eth_hdr->h_dest, "\x3c\xfd\xfe\x9e\x7f\x71", ETH_ALEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) memcpy(eth_hdr->h_source, "\xec\xb1\xd7\x98\x3a\xc0", ETH_ALEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) eth_hdr->h_proto = htons(ETH_P_IP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) /* IP header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) ip_hdr->version = IPVERSION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) ip_hdr->ihl = 0x5; /* 20 byte header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) ip_hdr->tos = 0x0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) ip_hdr->tot_len = htons(IP_PKT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) ip_hdr->id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) ip_hdr->frag_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) ip_hdr->ttl = IPDEFTTL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) ip_hdr->protocol = IPPROTO_UDP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) ip_hdr->saddr = htonl(0x0a0a0a10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) ip_hdr->daddr = htonl(0x0a0a0a20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) /* IP header checksum */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) ip_hdr->check = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) ip_hdr->check = ip_fast_csum((const void *)ip_hdr, ip_hdr->ihl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) /* UDP header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) udp_hdr->source = htons(0x1000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) udp_hdr->dest = htons(0x1000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) udp_hdr->len = htons(UDP_PKT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) /* UDP data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) memset32_htonl(pkt_data + PKT_HDR_SIZE, opt_pkt_fill_pattern,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) UDP_PKT_DATA_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) /* UDP header checksum */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) udp_hdr->check = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) udp_hdr->check = udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) IPPROTO_UDP, (u16 *)udp_hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) static void gen_eth_frame(struct xsk_umem_info *umem, u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) PKT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) struct xsk_umem_info *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) struct xsk_umem_config cfg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) /* We recommend that you set the fill ring size >= HW RX ring size +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * AF_XDP RX ring size. Make sure you fill up the fill ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) * with buffers at regular intervals, and you will with this setting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * avoid allocation failures in the driver. These are usually quite
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * expensive since drivers have not been written to assume that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * allocation failures are common. For regular sockets, kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * allocated memory is used that only runs out in OOM situations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * that should be rare.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS * 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) .frame_size = opt_xsk_frame_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) .flags = opt_umem_flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) umem = calloc(1, sizeof(*umem));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) if (!umem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) exit_with_error(errno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) &cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) exit_with_error(-ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) umem->buffer = buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) return umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) static void xsk_populate_fill_ring(struct xsk_umem_info *umem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) int ret, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) ret = xsk_ring_prod__reserve(&umem->fq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) XSK_RING_PROD__DEFAULT_NUM_DESCS * 2, &idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS * 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) exit_with_error(-ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS * 2; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) *xsk_ring_prod__fill_addr(&umem->fq, idx++) =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) i * opt_xsk_frame_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS * 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) bool rx, bool tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) struct xsk_socket_config cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) struct xsk_socket_info *xsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) struct xsk_ring_cons *rxr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) struct xsk_ring_prod *txr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) xsk = calloc(1, sizeof(*xsk));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) if (!xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) exit_with_error(errno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) xsk->umem = umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (opt_num_xsks > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) cfg.libbpf_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) cfg.xdp_flags = opt_xdp_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) cfg.bind_flags = opt_xdp_bind_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) rxr = rx ? &xsk->rx : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) txr = tx ? &xsk->tx : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) rxr, txr, &cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) exit_with_error(-ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) exit_with_error(-ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) xsk->app_stats.rx_empty_polls = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) xsk->app_stats.fill_fail_polls = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) xsk->app_stats.copy_tx_sendtos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) xsk->app_stats.tx_wakeup_sendtos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) xsk->app_stats.opt_polls = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) xsk->app_stats.prev_rx_empty_polls = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) xsk->app_stats.prev_fill_fail_polls = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) xsk->app_stats.prev_copy_tx_sendtos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) xsk->app_stats.prev_tx_wakeup_sendtos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) xsk->app_stats.prev_opt_polls = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) return xsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) static struct option long_options[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) {"rxdrop", no_argument, 0, 'r'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) {"txonly", no_argument, 0, 't'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) {"l2fwd", no_argument, 0, 'l'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) {"interface", required_argument, 0, 'i'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) {"queue", required_argument, 0, 'q'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) {"poll", no_argument, 0, 'p'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) {"xdp-skb", no_argument, 0, 'S'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) {"xdp-native", no_argument, 0, 'N'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) {"interval", required_argument, 0, 'n'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) {"zero-copy", no_argument, 0, 'z'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) {"copy", no_argument, 0, 'c'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) {"frame-size", required_argument, 0, 'f'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) {"no-need-wakeup", no_argument, 0, 'm'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) {"unaligned", no_argument, 0, 'u'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) {"shared-umem", no_argument, 0, 'M'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) {"force", no_argument, 0, 'F'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) {"duration", required_argument, 0, 'd'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) {"batch-size", required_argument, 0, 'b'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) {"tx-pkt-count", required_argument, 0, 'C'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) {"tx-pkt-size", required_argument, 0, 's'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) {"tx-pkt-pattern", required_argument, 0, 'P'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) {"extra-stats", no_argument, 0, 'x'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) {"quiet", no_argument, 0, 'Q'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) {"app-stats", no_argument, 0, 'a'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) {"irq-string", no_argument, 0, 'I'},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) {0, 0, 0, 0}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) static void usage(const char *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) const char *str =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) " Usage: %s [OPTIONS]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) " Options:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) " -r, --rxdrop Discard all incoming packets (default)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) " -t, --txonly Only send packets\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) " -l, --l2fwd MAC swap L2 forwarding\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) " -i, --interface=n Run on interface n\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) " -q, --queue=n Use queue n (default 0)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) " -p, --poll Use poll syscall\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) " -S, --xdp-skb=n Use XDP skb-mod\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) " -N, --xdp-native=n Enforce XDP native mode\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) " -n, --interval=n Specify statistics update interval (default 1 sec).\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) " -z, --zero-copy Force zero-copy mode.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) " -c, --copy Force copy mode.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) " -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) " -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) " -u, --unaligned Enable unaligned chunk placement\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) " -M, --shared-umem Enable XDP_SHARED_UMEM\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) " -F, --force Force loading the XDP prog\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) " -d, --duration=n Duration in secs to run command.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) " Default: forever.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) " -b, --batch-size=n Batch size for sending or receiving\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) " packets. Default: %d\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) " -C, --tx-pkt-count=n Number of packets to send.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) " Default: Continuous packets.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) " -s, --tx-pkt-size=n Transmit packet size.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) " (Default: %d bytes)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) " Min size: %d, Max size %d.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) " -P, --tx-pkt-pattern=nPacket fill pattern. Default: 0x%x\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) " -x, --extra-stats Display extra statistics.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) " -Q, --quiet Do not display any stats.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) " -a, --app-stats Display application (syscall) statistics.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) " -I, --irq-string Display driver interrupt statistics for interface associated with irq-string.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) "\n";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) opt_batch_size, MIN_PKT_SIZE, MIN_PKT_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) XSK_UMEM__DEFAULT_FRAME_SIZE, opt_pkt_fill_pattern);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) static void parse_command_line(int argc, char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) int option_index, c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) opterr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:s:P:xQaI:",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) long_options, &option_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) if (c == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) switch (c) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) case 'r':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) opt_bench = BENCH_RXDROP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) case 't':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) opt_bench = BENCH_TXONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) case 'l':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) opt_bench = BENCH_L2FWD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) case 'i':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) opt_if = optarg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) case 'q':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) opt_queue = atoi(optarg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) case 'p':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) opt_poll = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) case 'S':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) opt_xdp_flags |= XDP_FLAGS_SKB_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) opt_xdp_bind_flags |= XDP_COPY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) case 'N':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) /* default, set below */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) case 'n':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) opt_interval = atoi(optarg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) case 'z':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) opt_xdp_bind_flags |= XDP_ZEROCOPY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) case 'c':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) opt_xdp_bind_flags |= XDP_COPY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) case 'u':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) opt_umem_flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) opt_unaligned_chunks = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) opt_mmap_flags = MAP_HUGETLB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) case 'F':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) case 'f':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) opt_xsk_frame_size = atoi(optarg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) case 'm':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) opt_need_wakeup = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) opt_xdp_bind_flags &= ~XDP_USE_NEED_WAKEUP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) case 'M':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) opt_num_xsks = MAX_SOCKS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) case 'd':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) opt_duration = atoi(optarg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) opt_duration *= 1000000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) case 'b':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) opt_batch_size = atoi(optarg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) case 'C':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) opt_pkt_count = atoi(optarg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) case 's':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) opt_pkt_size = atoi(optarg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) if (opt_pkt_size > (XSK_UMEM__DEFAULT_FRAME_SIZE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) opt_pkt_size < MIN_PKT_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) fprintf(stderr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) "ERROR: Invalid frame size %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) opt_pkt_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) usage(basename(argv[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) case 'P':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) opt_pkt_fill_pattern = strtol(optarg, NULL, 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) case 'x':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) opt_extra_stats = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) case 'Q':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) opt_quiet = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) case 'a':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) opt_app_stats = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) case 'I':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) opt_irq_str = optarg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) if (get_interrupt_number())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) irqs_at_init = get_irqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) if (irqs_at_init < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) fprintf(stderr, "ERROR: Failed to get irqs for %s\n", opt_irq_str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) usage(basename(argv[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) usage(basename(argv[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) if (!(opt_xdp_flags & XDP_FLAGS_SKB_MODE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) opt_xdp_flags |= XDP_FLAGS_DRV_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) opt_ifindex = if_nametoindex(opt_if);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) if (!opt_ifindex) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) fprintf(stderr, "ERROR: interface \"%s\" does not exist\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) opt_if);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) usage(basename(argv[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) if ((opt_xsk_frame_size & (opt_xsk_frame_size - 1)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) !opt_unaligned_chunks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) fprintf(stderr, "--frame-size=%d is not a power of two\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) opt_xsk_frame_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) usage(basename(argv[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) static void kick_tx(struct xsk_socket_info *xsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) errno == EBUSY || errno == ENETDOWN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) exit_with_error(errno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) struct pollfd *fds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) struct xsk_umem_info *umem = xsk->umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) u32 idx_cq = 0, idx_fq = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) unsigned int rcvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) size_t ndescs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) if (!xsk->outstanding_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) /* In copy mode, Tx is driven by a syscall so we need to use e.g. sendto() to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) * really send the packets. In zero-copy mode we do not have to do this, since Tx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) * is driven by the NAPI loop. So as an optimization, we do not have to call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) * sendto() all the time in zero-copy mode for l2fwd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) if (opt_xdp_bind_flags & XDP_COPY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) xsk->app_stats.copy_tx_sendtos++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) kick_tx(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) ndescs = (xsk->outstanding_tx > opt_batch_size) ? opt_batch_size :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) xsk->outstanding_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) /* re-add completed Tx buffers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) rcvd = xsk_ring_cons__peek(&umem->cq, ndescs, &idx_cq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) if (rcvd > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) while (ret != rcvd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) exit_with_error(-ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) xsk->app_stats.fill_fail_polls++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) ret = poll(fds, num_socks, opt_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) for (i = 0; i < rcvd; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) *xsk_ring_cons__comp_addr(&umem->cq, idx_cq++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) xsk_ring_cons__release(&xsk->umem->cq, rcvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) xsk->outstanding_tx -= rcvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) xsk->ring_stats.tx_npkts += rcvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) static inline void complete_tx_only(struct xsk_socket_info *xsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) int batch_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) unsigned int rcvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) if (!xsk->outstanding_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) xsk->app_stats.tx_wakeup_sendtos++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) kick_tx(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) if (rcvd > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) xsk_ring_cons__release(&xsk->umem->cq, rcvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) xsk->outstanding_tx -= rcvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) xsk->ring_stats.tx_npkts += rcvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) unsigned int rcvd, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) u32 idx_rx = 0, idx_fq = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) if (!rcvd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) xsk->app_stats.rx_empty_polls++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) ret = poll(fds, num_socks, opt_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) while (ret != rcvd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) exit_with_error(-ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) xsk->app_stats.fill_fail_polls++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) ret = poll(fds, num_socks, opt_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) for (i = 0; i < rcvd; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) u64 orig = xsk_umem__extract_addr(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) addr = xsk_umem__add_offset_to_addr(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) hex_dump(pkt, len, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) xsk_ring_cons__release(&xsk->rx, rcvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) xsk->ring_stats.rx_npkts += rcvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) static void rx_drop_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) struct pollfd fds[MAX_SOCKS] = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) for (i = 0; i < num_socks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) fds[i].fd = xsk_socket__fd(xsks[i]->xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) fds[i].events = POLLIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) if (opt_poll) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) for (i = 0; i < num_socks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) xsks[i]->app_stats.opt_polls++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) ret = poll(fds, num_socks, opt_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) for (i = 0; i < num_socks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) rx_drop(xsks[i], fds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) if (benchmark_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) static void tx_only(struct xsk_socket_info *xsk, u32 *frame_nb, int batch_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) while (xsk_ring_prod__reserve(&xsk->tx, batch_size, &idx) <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) batch_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) complete_tx_only(xsk, batch_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) if (benchmark_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) for (i = 0; i < batch_size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) idx + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) tx_desc->addr = (*frame_nb + i) * opt_xsk_frame_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) tx_desc->len = PKT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) xsk_ring_prod__submit(&xsk->tx, batch_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) xsk->outstanding_tx += batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) *frame_nb += batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) *frame_nb %= NUM_FRAMES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) complete_tx_only(xsk, batch_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) static inline int get_batch_size(int pkt_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) if (!opt_pkt_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) return opt_batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if (pkt_cnt + opt_batch_size <= opt_pkt_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) return opt_batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) return opt_pkt_count - pkt_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) static void complete_tx_only_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) bool pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) pending = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) for (i = 0; i < num_socks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) if (xsks[i]->outstanding_tx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) complete_tx_only(xsks[i], opt_batch_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) pending = !!xsks[i]->outstanding_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) } while (pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) static void tx_only_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) struct pollfd fds[MAX_SOCKS] = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) u32 frame_nb[MAX_SOCKS] = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) int pkt_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) for (i = 0; i < num_socks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) fds[0].fd = xsk_socket__fd(xsks[i]->xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) fds[0].events = POLLOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) while ((opt_pkt_count && pkt_cnt < opt_pkt_count) || !opt_pkt_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) int batch_size = get_batch_size(pkt_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) if (opt_poll) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) for (i = 0; i < num_socks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) xsks[i]->app_stats.opt_polls++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) ret = poll(fds, num_socks, opt_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) if (!(fds[0].revents & POLLOUT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) for (i = 0; i < num_socks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) tx_only(xsks[i], &frame_nb[i], batch_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) pkt_cnt += batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) if (benchmark_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) if (opt_pkt_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) complete_tx_only_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) unsigned int rcvd, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) u32 idx_rx = 0, idx_tx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) complete_tx_l2fwd(xsk, fds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) if (!rcvd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) xsk->app_stats.rx_empty_polls++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) ret = poll(fds, num_socks, opt_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) while (ret != rcvd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) exit_with_error(-ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) complete_tx_l2fwd(xsk, fds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) xsk->app_stats.tx_wakeup_sendtos++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) kick_tx(xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) for (i = 0; i < rcvd; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) u64 orig = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) addr = xsk_umem__add_offset_to_addr(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) swap_mac_addresses(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) hex_dump(pkt, len, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) xsk_ring_prod__submit(&xsk->tx, rcvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) xsk_ring_cons__release(&xsk->rx, rcvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) xsk->ring_stats.rx_npkts += rcvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) xsk->outstanding_tx += rcvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) static void l2fwd_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) struct pollfd fds[MAX_SOCKS] = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) for (i = 0; i < num_socks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) fds[i].fd = xsk_socket__fd(xsks[i]->xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) fds[i].events = POLLOUT | POLLIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) if (opt_poll) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) for (i = 0; i < num_socks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) xsks[i]->app_stats.opt_polls++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) ret = poll(fds, num_socks, opt_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) for (i = 0; i < num_socks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) l2fwd(xsks[i], fds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) if (benchmark_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) static void load_xdp_program(char **argv, struct bpf_object **obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) struct bpf_prog_load_attr prog_load_attr = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) .prog_type = BPF_PROG_TYPE_XDP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) char xdp_filename[256];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) int prog_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) prog_load_attr.file = xdp_filename;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) if (bpf_prog_load_xattr(&prog_load_attr, obj, &prog_fd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) if (prog_fd < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) fprintf(stderr, "ERROR: no program found: %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) strerror(prog_fd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) if (bpf_set_link_xdp_fd(opt_ifindex, prog_fd, opt_xdp_flags) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) fprintf(stderr, "ERROR: link set xdp fd failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) static void enter_xsks_into_map(struct bpf_object *obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) struct bpf_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) int i, xsks_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) map = bpf_object__find_map_by_name(obj, "xsks_map");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) xsks_map = bpf_map__fd(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) if (xsks_map < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) fprintf(stderr, "ERROR: no xsks map found: %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) strerror(xsks_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) for (i = 0; i < num_socks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) int fd = xsk_socket__fd(xsks[i]->xsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) int key, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) key = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) ret = bpf_map_update_elem(xsks_map, &key, &fd, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) fprintf(stderr, "ERROR: bpf_map_update_elem %d\n", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) int main(int argc, char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) bool rx = false, tx = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) struct xsk_umem_info *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) struct bpf_object *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) pthread_t pt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) void *bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) parse_command_line(argc, argv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) if (setrlimit(RLIMIT_MEMLOCK, &r)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) strerror(errno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) if (opt_num_xsks > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) load_xdp_program(argv, &obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) /* Reserve memory for the umem. Use hugepages if unaligned chunk mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) bufs = mmap(NULL, NUM_FRAMES * opt_xsk_frame_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) PROT_READ | PROT_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) MAP_PRIVATE | MAP_ANONYMOUS | opt_mmap_flags, -1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) if (bufs == MAP_FAILED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) printf("ERROR: mmap failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) exit(EXIT_FAILURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) /* Create sockets... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) if (opt_bench == BENCH_RXDROP || opt_bench == BENCH_L2FWD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) rx = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) xsk_populate_fill_ring(umem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) if (opt_bench == BENCH_L2FWD || opt_bench == BENCH_TXONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) tx = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) for (i = 0; i < opt_num_xsks; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) xsks[num_socks++] = xsk_configure_socket(umem, rx, tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) if (opt_bench == BENCH_TXONLY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) gen_eth_hdr_data();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) for (i = 0; i < NUM_FRAMES; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) gen_eth_frame(umem, i * opt_xsk_frame_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) if (opt_num_xsks > 1 && opt_bench != BENCH_TXONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) enter_xsks_into_map(obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) signal(SIGINT, int_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) signal(SIGTERM, int_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) signal(SIGABRT, int_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) setlocale(LC_ALL, "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) prev_time = get_nsecs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) start_time = prev_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) if (!opt_quiet) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) ret = pthread_create(&pt, NULL, poller, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) exit_with_error(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) if (opt_bench == BENCH_RXDROP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) rx_drop_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) else if (opt_bench == BENCH_TXONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) tx_only_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) l2fwd_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) benchmark_done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) if (!opt_quiet)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) pthread_join(pt, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) xdpsock_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) munmap(bufs, NUM_FRAMES * opt_xsk_frame_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags