^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * TUN - Universal TUN/TAP device driver.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Changes:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Add TUNSETLINK ioctl to set the link encapsulation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Mark Smith <markzzzsmith@yahoo.com.au>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * Use eth_random_addr() for tap MAC address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * Harald Roelle <harald.roelle@ifi.lmu.de> 2004/04/20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * Fixes in packet dropping, queue length setting and queue wakeup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Increased default tx queue length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * Added ethtool API.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * Minor cleanups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * Daniel Podlejski <underley@underley.eu.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * Modifications for 2.3.99-pre5 kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define DRV_NAME "tun"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define DRV_VERSION "1.6"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define DRV_DESCRIPTION "Universal TUN/TAP device driver"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define DRV_COPYRIGHT "(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/major.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/poll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/fcntl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/skbuff.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/netdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/etherdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/miscdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/ethtool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/rtnetlink.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/compat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/if.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/if_arp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/if_ether.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/if_tun.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <linux/if_vlan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <linux/crc32.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <linux/nsproxy.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <linux/virtio_net.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include <net/net_namespace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include <net/netns/generic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #include <net/rtnetlink.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include <net/sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include <net/xdp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include <net/ip_tunnels.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #include <linux/uio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #include <linux/skb_array.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #include <linux/bpf_trace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #include <linux/ieee802154.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #include <linux/if_ltalk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #include <uapi/linux/if_fddi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #include <uapi/linux/if_hippi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #include <uapi/linux/if_fc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #include <net/ax25.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #include <net/rose.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #include <net/6lowpan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #include <linux/proc_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static void tun_default_link_ksettings(struct net_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct ethtool_link_ksettings *cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) /* TUN device flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) /* IFF_ATTACH_QUEUE is never stored in device flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * overload it to mean fasync when stored there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define TUN_FASYNC IFF_ATTACH_QUEUE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) /* High bits in flags field are unused. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define TUN_VNET_LE 0x80000000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define TUN_VNET_BE 0x40000000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define GOODCOPY_LEN 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #define FLT_EXACT_COUNT 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct tap_filter {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) unsigned int count; /* Number of addrs. Zero means disabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) u32 mask[2]; /* Mask of the hashed addrs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) unsigned char addr[FLT_EXACT_COUNT][ETH_ALEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * to max number of VCPUs in guest. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define MAX_TAP_QUEUES 256
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #define MAX_TAP_FLOWS 4096
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define TUN_FLOW_EXPIRE (3 * HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct tun_pcpu_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) u64_stats_t rx_packets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) u64_stats_t rx_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) u64_stats_t tx_packets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) u64_stats_t tx_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct u64_stats_sync syncp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) u32 rx_dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) u32 tx_dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) u32 rx_frame_errors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /* A tun_file connects an open character device to a tuntap netdevice. It
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * also contains all socket related structures (except sock_fprog and tap_filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * to serve as one transmit queue for tuntap device. The sock_fprog and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * tap_filter were kept in tun_struct since they were used for filtering for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * netdevice not for a specific queue (at least I didn't see the requirement for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * this).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * RCU usage:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * The tun_file and tun_struct are loosely coupled, the pointer from one to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * other can only be read while rcu_read_lock or rtnl_lock is held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct tun_file {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct sock sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) struct socket socket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct tun_struct __rcu *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) struct fasync_struct *fasync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) /* only used for fasnyc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) unsigned int flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) u16 queue_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) unsigned int ifindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) struct napi_struct napi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) bool napi_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) bool napi_frags_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) struct mutex napi_mutex; /* Protects access to the above napi */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) struct list_head next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) struct tun_struct *detached;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) struct ptr_ring tx_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct xdp_rxq_info xdp_rxq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) struct tun_page {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct tun_flow_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct hlist_node hash_link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) struct tun_struct *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) u32 rxhash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) u32 rps_rxhash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) int queue_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) unsigned long updated ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #define TUN_NUM_FLOW_ENTRIES 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) struct tun_prog {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) struct bpf_prog *prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /* Since the socket were moved to tun_file, to preserve the behavior of persist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * device, socket filter, sndbuf and vnet header size were restore when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * file were attached to a persist device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct tun_struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) struct tun_file __rcu *tfiles[MAX_TAP_QUEUES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) unsigned int numqueues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) unsigned int flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) kuid_t owner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) kgid_t group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) struct net_device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) netdev_features_t set_features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) #define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) NETIF_F_TSO6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) int align;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) int vnet_hdr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) int sndbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) struct tap_filter txflt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) struct sock_fprog fprog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) /* protected by rtnl lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) bool filter_attached;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) u32 msg_enable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct hlist_head flows[TUN_NUM_FLOW_ENTRIES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) struct timer_list flow_gc_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) unsigned long ageing_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) unsigned int numdisabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct list_head disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) void *security;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) u32 flow_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) u32 rx_batched;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct tun_pcpu_stats __percpu *pcpu_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) struct bpf_prog __rcu *xdp_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) struct tun_prog __rcu *steering_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct tun_prog __rcu *filter_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) struct ethtool_link_ksettings link_ksettings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct veth {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) __be16 h_vlan_proto;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) __be16 h_vlan_TCI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static int tun_napi_receive(struct napi_struct *napi, int budget)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct tun_file *tfile = container_of(napi, struct tun_file, napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct sk_buff_head process_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) int received = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) __skb_queue_head_init(&process_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) spin_lock(&queue->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) skb_queue_splice_tail_init(queue, &process_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) spin_unlock(&queue->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) while (received < budget && (skb = __skb_dequeue(&process_queue))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) napi_gro_receive(napi, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) ++received;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (!skb_queue_empty(&process_queue)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) spin_lock(&queue->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) skb_queue_splice(&process_queue, queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) spin_unlock(&queue->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) return received;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) static int tun_napi_poll(struct napi_struct *napi, int budget)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) unsigned int received;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) received = tun_napi_receive(napi, budget);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (received < budget)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) napi_complete_done(napi, received);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) return received;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) bool napi_en, bool napi_frags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) tfile->napi_enabled = napi_en;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) tfile->napi_frags_enabled = napi_en && napi_frags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (napi_en) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) netif_tx_napi_add(tun->dev, &tfile->napi, tun_napi_poll,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) NAPI_POLL_WEIGHT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) napi_enable(&tfile->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) static void tun_napi_disable(struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (tfile->napi_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) napi_disable(&tfile->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static void tun_napi_del(struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (tfile->napi_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) netif_napi_del(&tfile->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static bool tun_napi_frags_enabled(const struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) return tfile->napi_frags_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) #ifdef CONFIG_TUN_VNET_CROSS_LE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) return tun->flags & TUN_VNET_BE ? false :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) virtio_legacy_is_little_endian();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) int be = !!(tun->flags & TUN_VNET_BE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (put_user(be, argp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) int be;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (get_user(be, argp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (be)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) tun->flags |= TUN_VNET_BE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) tun->flags &= ~TUN_VNET_BE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return virtio_legacy_is_little_endian();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) #endif /* CONFIG_TUN_VNET_CROSS_LE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static inline bool tun_is_little_endian(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) return tun->flags & TUN_VNET_LE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) tun_legacy_is_little_endian(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return __virtio16_to_cpu(tun_is_little_endian(tun), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return __cpu_to_virtio16(tun_is_little_endian(tun), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) static inline u32 tun_hashfn(u32 rxhash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return rxhash & TUN_MASK_FLOW_ENTRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct tun_flow_entry *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) hlist_for_each_entry_rcu(e, head, hash_link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) if (e->rxhash == rxhash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) return e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) struct hlist_head *head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) u32 rxhash, u16 queue_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (e) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) netif_info(tun, tx_queued, tun->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) "create flow: hash %u index %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) rxhash, queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) e->updated = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) e->rxhash = rxhash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) e->rps_rxhash = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) e->queue_index = queue_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) e->tun = tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) hlist_add_head_rcu(&e->hash_link, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) ++tun->flow_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) return e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) netif_info(tun, tx_queued, tun->dev, "delete flow: hash %u index %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) e->rxhash, e->queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) hlist_del_rcu(&e->hash_link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) kfree_rcu(e, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) --tun->flow_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) static void tun_flow_flush(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) spin_lock_bh(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct tun_flow_entry *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) struct hlist_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) tun_flow_delete(tun, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) spin_unlock_bh(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) spin_lock_bh(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) struct tun_flow_entry *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) struct hlist_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (e->queue_index == queue_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) tun_flow_delete(tun, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) spin_unlock_bh(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) static void tun_flow_cleanup(struct timer_list *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) struct tun_struct *tun = from_timer(tun, t, flow_gc_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) unsigned long delay = tun->ageing_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) unsigned long next_timer = jiffies + delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) unsigned long count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) spin_lock(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) struct tun_flow_entry *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) struct hlist_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) unsigned long this_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) this_timer = e->updated + delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) if (time_before_eq(this_timer, jiffies)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) tun_flow_delete(tun, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) if (time_before(this_timer, next_timer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) next_timer = this_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) spin_unlock(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) struct hlist_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) struct tun_flow_entry *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) unsigned long delay = tun->ageing_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) u16 queue_index = tfile->queue_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) head = &tun->flows[tun_hashfn(rxhash)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) e = tun_flow_find(head, rxhash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (likely(e)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) /* TODO: keep queueing to old queue until it's empty? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (READ_ONCE(e->queue_index) != queue_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) WRITE_ONCE(e->queue_index, queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (e->updated != jiffies)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) e->updated = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) sock_rps_record_flow_hash(e->rps_rxhash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) spin_lock_bh(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) if (!tun_flow_find(head, rxhash) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) tun->flow_count < MAX_TAP_FLOWS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) tun_flow_create(tun, head, rxhash, queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (!timer_pending(&tun->flow_gc_timer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) mod_timer(&tun->flow_gc_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) round_jiffies_up(jiffies + delay));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) spin_unlock_bh(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) /* Save the hash received in the stack receive path and update the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) * flow_hash table accordingly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (unlikely(e->rps_rxhash != hash))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) e->rps_rxhash = hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /* We try to identify a flow through its rxhash. The reason that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * we do not check rxq no. is because some cards(e.g 82599), chooses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * the rxq based on the txq where the last packet of the flow comes. As
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * the userspace application move between processors, we may get a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * different rxq no. here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) struct tun_flow_entry *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) u32 txq = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) u32 numqueues = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) numqueues = READ_ONCE(tun->numqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) txq = __skb_get_hash_symmetric(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (e) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) tun_flow_save_rps_rxhash(e, txq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) txq = e->queue_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) /* use multiply and shift instead of expensive divide */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) txq = ((u64)txq * numqueues) >> 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) return txq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) struct tun_prog *prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) u32 numqueues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) u16 ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) numqueues = READ_ONCE(tun->numqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) if (!numqueues)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) prog = rcu_dereference(tun->steering_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) ret = bpf_prog_run_clear_cb(prog->prog, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) return ret % numqueues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) struct net_device *sb_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) u16 ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) if (rcu_dereference(tun->steering_prog))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) ret = tun_ebpf_select_queue(tun, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) ret = tun_automq_select_queue(tun, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) static inline bool tun_not_capable(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) const struct cred *cred = current_cred();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) struct net *net = dev_net(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) (gid_valid(tun->group) && !in_egroup_p(tun->group))) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) !ns_capable(net->user_ns, CAP_NET_ADMIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) static void tun_set_real_num_queues(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) netif_set_real_num_tx_queues(tun->dev, tun->numqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) netif_set_real_num_rx_queues(tun->dev, tun->numqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) tfile->detached = tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) list_add_tail(&tfile->next, &tun->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) ++tun->numdisabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) struct tun_struct *tun = tfile->detached;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) tfile->detached = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) list_del_init(&tfile->next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) --tun->numdisabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) return tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) void tun_ptr_free(void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) if (!ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if (tun_is_xdp_frame(ptr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) xdp_return_frame(xdpf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) __skb_array_destroy_skb(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) EXPORT_SYMBOL_GPL(tun_ptr_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) static void tun_queue_purge(struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) void *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) tun_ptr_free(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) skb_queue_purge(&tfile->sk.sk_write_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) skb_queue_purge(&tfile->sk.sk_error_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) static void __tun_detach(struct tun_file *tfile, bool clean)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) struct tun_file *ntfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) struct tun_struct *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) tun = rtnl_dereference(tfile->tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (tun && clean) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) tun_napi_disable(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) tun_napi_del(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (tun && !tfile->detached) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) u16 index = tfile->queue_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) BUG_ON(index >= tun->numqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) rcu_assign_pointer(tun->tfiles[index],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) tun->tfiles[tun->numqueues - 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) ntfile = rtnl_dereference(tun->tfiles[index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) ntfile->queue_index = index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) --tun->numqueues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) if (clean) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) RCU_INIT_POINTER(tfile->tun, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) sock_put(&tfile->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) tun_disable_queue(tun, tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) synchronize_net();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) tun_flow_delete_by_queue(tun, tun->numqueues + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) /* Drop read queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) tun_queue_purge(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) tun_set_real_num_queues(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) } else if (tfile->detached && clean) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) tun = tun_enable_queue(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) sock_put(&tfile->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) if (clean) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) if (tun && tun->numqueues == 0 && tun->numdisabled == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) netif_carrier_off(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) if (!(tun->flags & IFF_PERSIST) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) tun->dev->reg_state == NETREG_REGISTERED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) unregister_netdevice(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) if (tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) xdp_rxq_info_unreg(&tfile->xdp_rxq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) sock_put(&tfile->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) static void tun_detach(struct tun_file *tfile, bool clean)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) struct tun_struct *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) struct net_device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) rtnl_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) tun = rtnl_dereference(tfile->tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) dev = tun ? tun->dev : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) __tun_detach(tfile, clean);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) if (dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) netdev_state_change(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) rtnl_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) static void tun_detach_all(struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) struct tun_file *tfile, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) int i, n = tun->numqueues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) for (i = 0; i < n; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) tfile = rtnl_dereference(tun->tfiles[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) BUG_ON(!tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) tun_napi_disable(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) tfile->socket.sk->sk_data_ready(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) RCU_INIT_POINTER(tfile->tun, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) --tun->numqueues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) list_for_each_entry(tfile, &tun->disabled, next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) tfile->socket.sk->sk_data_ready(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) RCU_INIT_POINTER(tfile->tun, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) BUG_ON(tun->numqueues != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) synchronize_net();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) for (i = 0; i < n; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) tfile = rtnl_dereference(tun->tfiles[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) tun_napi_del(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) /* Drop read queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) tun_queue_purge(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) xdp_rxq_info_unreg(&tfile->xdp_rxq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) sock_put(&tfile->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) tun_enable_queue(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) tun_queue_purge(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) xdp_rxq_info_unreg(&tfile->xdp_rxq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) sock_put(&tfile->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) BUG_ON(tun->numdisabled != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) if (tun->flags & IFF_PERSIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) module_put(THIS_MODULE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) static int tun_attach(struct tun_struct *tun, struct file *file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) bool skip_filter, bool napi, bool napi_frags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) bool publish_tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) struct net_device *dev = tun->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) err = security_tun_dev_attach(tfile->socket.sk, tun->security);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) if (rtnl_dereference(tfile->tun) && !tfile->detached)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) err = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (!tfile->detached &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) /* Re-attach the filter to persist device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) if (!skip_filter && (tun->filter_attached == true)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) lock_sock(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) err = sk_attach_filter(&tun->fprog, tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) release_sock(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (!tfile->detached &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) GFP_KERNEL, tun_ptr_free)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) tfile->queue_index = tun->numqueues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (tfile->detached) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) /* Re-attach detached tfile, updating XDP queue_index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (tfile->xdp_rxq.queue_index != tfile->queue_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) tfile->xdp_rxq.queue_index = tfile->queue_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) /* Setup XDP RX-queue info, for new tfile getting attached */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) err = xdp_rxq_info_reg(&tfile->xdp_rxq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) tun->dev, tfile->queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) MEM_TYPE_PAGE_SHARED, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) xdp_rxq_info_unreg(&tfile->xdp_rxq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) if (tfile->detached) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) tun_enable_queue(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) sock_hold(&tfile->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) tun_napi_init(tun, tfile, napi, napi_frags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (rtnl_dereference(tun->xdp_prog))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) sock_set_flag(&tfile->sk, SOCK_XDP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) /* device is allowed to go away first, so no need to hold extra
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) * refcnt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) /* Publish tfile->tun and tun->tfiles only after we've fully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) * initialized tfile; otherwise we risk using half-initialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) * object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) if (publish_tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) rcu_assign_pointer(tfile->tun, tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) tun->numqueues++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) tun_set_real_num_queues(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) static struct tun_struct *tun_get(struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) struct tun_struct *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) tun = rcu_dereference(tfile->tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) dev_hold(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) return tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) static void tun_put(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) dev_put(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) /* TAP filtering */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) static void addr_hash_set(u32 *mask, const u8 *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) int n = ether_crc(ETH_ALEN, addr) >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) mask[n >> 5] |= (1 << (n & 31));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) static unsigned int addr_hash_test(const u32 *mask, const u8 *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) int n = ether_crc(ETH_ALEN, addr) >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) return mask[n >> 5] & (1 << (n & 31));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) static int update_filter(struct tap_filter *filter, void __user *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) struct { u8 u[ETH_ALEN]; } *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) struct tun_filter uf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) int err, alen, n, nexact;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) if (copy_from_user(&uf, arg, sizeof(uf)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) if (!uf.count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) /* Disabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) filter->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) alen = ETH_ALEN * uf.count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) addr = memdup_user(arg + sizeof(uf), alen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (IS_ERR(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) return PTR_ERR(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) /* The filter is updated without holding any locks. Which is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * perfectly safe. We disable it first and in the worst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * case we'll accept a few undesired packets. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) filter->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) /* Use first set of addresses as an exact filter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) memcpy(filter->addr[n], addr[n].u, ETH_ALEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) nexact = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) /* Remaining multicast addresses are hashed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * unicast will leave the filter disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) memset(filter->mask, 0, sizeof(filter->mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) for (; n < uf.count; n++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (!is_multicast_ether_addr(addr[n].u)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) err = 0; /* no filter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) goto free_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) addr_hash_set(filter->mask, addr[n].u);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) /* For ALLMULTI just set the mask to all ones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * This overrides the mask populated above. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) if ((uf.flags & TUN_FLT_ALLMULTI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) memset(filter->mask, ~0, sizeof(filter->mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) /* Now enable the filter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) filter->count = nexact;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) /* Return the number of exact filters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) err = nexact;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) free_addr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) kfree(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) /* Returns: 0 - drop, !=0 - accept */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) static int run_filter(struct tap_filter *filter, const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) /* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) * at this point. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) struct ethhdr *eh = (struct ethhdr *) skb->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) /* Exact match */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) for (i = 0; i < filter->count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) if (ether_addr_equal(eh->h_dest, filter->addr[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) /* Inexact match (multicast only) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) if (is_multicast_ether_addr(eh->h_dest))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) return addr_hash_test(filter->mask, eh->h_dest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * Checks whether the packet is accepted or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * Returns: 0 - drop, !=0 - accept
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) static int check_filter(struct tap_filter *filter, const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) if (!filter->count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return run_filter(filter, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) /* Network device part of the driver */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) static const struct ethtool_ops tun_ethtool_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) /* Net device detach from fd. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) static void tun_net_uninit(struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) tun_detach_all(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) /* Net device open. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) static int tun_net_open(struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) netif_tx_start_all_queues(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) /* Net device close. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) static int tun_net_close(struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) netif_tx_stop_all_queues(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) /* Net device start xmit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) #ifdef CONFIG_RPS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) /* Select queue was not called for the skbuff, so we extract the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) * RPS hash and save it into the flow_table here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) struct tun_flow_entry *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) __u32 rxhash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) rxhash = __skb_get_hash_symmetric(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) if (e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) tun_flow_save_rps_rxhash(e, rxhash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) static unsigned int run_ebpf_filter(struct tun_struct *tun,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) struct tun_prog *prog = rcu_dereference(tun->filter_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) if (prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) len = bpf_prog_run_clear_cb(prog->prog, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) /* Net device start xmit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) int txq = skb->queue_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) struct netdev_queue *queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) int len = skb->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) tfile = rcu_dereference(tun->tfiles[txq]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) /* Drop packet if interface is not attached */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) if (!tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) if (!rcu_dereference(tun->steering_prog))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) tun_automq_xmit(tun, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) netif_info(tun, tx_queued, tun->dev, "%s %d\n", __func__, skb->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) /* Drop if the filter does not like it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * This is a noop if the filter is disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * Filter can be enabled only for the TAP devices. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) if (!check_filter(&tun->txflt, skb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) if (tfile->socket.sk->sk_filter &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) sk_filter(tfile->socket.sk, skb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) len = run_ebpf_filter(tun, skb, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) if (len == 0 || pskb_trim(skb, len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) skb_tx_timestamp(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) /* Orphan the skb - required as we might hang on to it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) * for indefinite time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) skb_orphan(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) nf_reset_ct(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) if (ptr_ring_produce(&tfile->tx_ring, skb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) /* NETIF_F_LLTX requires to do our own update of trans_start */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) queue = netdev_get_tx_queue(dev, txq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) queue->trans_start = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) /* Notify and wake up reader process */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) if (tfile->flags & TUN_FASYNC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) tfile->socket.sk->sk_data_ready(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) return NETDEV_TX_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) drop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) this_cpu_inc(tun->pcpu_stats->tx_dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) skb_tx_error(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) return NET_XMIT_DROP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) static void tun_net_mclist(struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * This callback is supposed to deal with mc filter in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) * _rx_ path and has nothing to do with the _tx_ path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) * In rx path we always accept everything userspace gives us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) static netdev_features_t tun_net_fix_features(struct net_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) netdev_features_t features)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) return (features & tun->set_features) | (features & ~TUN_USER_FEATURES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) static void tun_set_headroom(struct net_device *dev, int new_hr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) if (new_hr < NET_SKB_PAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) new_hr = NET_SKB_PAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) tun->align = new_hr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) u32 rx_dropped = 0, tx_dropped = 0, rx_frame_errors = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) struct tun_pcpu_stats *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) for_each_possible_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) u64 rxpackets, rxbytes, txpackets, txbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) unsigned int start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) p = per_cpu_ptr(tun->pcpu_stats, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) start = u64_stats_fetch_begin(&p->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) rxpackets = u64_stats_read(&p->rx_packets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) rxbytes = u64_stats_read(&p->rx_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) txpackets = u64_stats_read(&p->tx_packets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) txbytes = u64_stats_read(&p->tx_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) } while (u64_stats_fetch_retry(&p->syncp, start));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) stats->rx_packets += rxpackets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) stats->rx_bytes += rxbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) stats->tx_packets += txpackets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) stats->tx_bytes += txbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) /* u32 counters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) rx_dropped += p->rx_dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) rx_frame_errors += p->rx_frame_errors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) tx_dropped += p->tx_dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) stats->rx_dropped = rx_dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) stats->rx_frame_errors = rx_frame_errors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) stats->tx_dropped = tx_dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) struct bpf_prog *old_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) old_prog = rtnl_dereference(tun->xdp_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) rcu_assign_pointer(tun->xdp_prog, prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) if (old_prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) bpf_prog_put(old_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) for (i = 0; i < tun->numqueues; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) tfile = rtnl_dereference(tun->tfiles[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) sock_set_flag(&tfile->sk, SOCK_XDP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) sock_reset_flag(&tfile->sk, SOCK_XDP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) list_for_each_entry(tfile, &tun->disabled, next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) if (prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) sock_set_flag(&tfile->sk, SOCK_XDP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) sock_reset_flag(&tfile->sk, SOCK_XDP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) switch (xdp->command) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) case XDP_SETUP_PROG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) return tun_xdp_set(dev, xdp->prog, xdp->extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) static int tun_net_change_carrier(struct net_device *dev, bool new_carrier)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (new_carrier) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) if (!tun->numqueues)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) netif_carrier_on(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) netif_carrier_off(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) static const struct net_device_ops tun_netdev_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) .ndo_uninit = tun_net_uninit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) .ndo_open = tun_net_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) .ndo_stop = tun_net_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) .ndo_start_xmit = tun_net_xmit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) .ndo_fix_features = tun_net_fix_features,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) .ndo_select_queue = tun_select_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) .ndo_set_rx_headroom = tun_set_headroom,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) .ndo_get_stats64 = tun_net_get_stats64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) .ndo_change_carrier = tun_net_change_carrier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) static void __tun_xdp_flush_tfile(struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) /* Notify and wake up reader process */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) if (tfile->flags & TUN_FASYNC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) tfile->socket.sk->sk_data_ready(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) static int tun_xdp_xmit(struct net_device *dev, int n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) struct xdp_frame **frames, u32 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) u32 numqueues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) int drops = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) int cnt = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) resample:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) numqueues = READ_ONCE(tun->numqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) if (!numqueues) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) return -ENXIO; /* Caller will free/return all frames */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) numqueues]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) if (unlikely(!tfile))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) goto resample;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) spin_lock(&tfile->tx_ring.producer_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) for (i = 0; i < n; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) struct xdp_frame *xdp = frames[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) /* Encode the XDP flag into lowest bit for consumer to differ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) * XDP buffer from sk_buff.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) void *frame = tun_xdp_to_ptr(xdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) if (__ptr_ring_produce(&tfile->tx_ring, frame)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) this_cpu_inc(tun->pcpu_stats->tx_dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) xdp_return_frame_rx_napi(xdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) drops++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) spin_unlock(&tfile->tx_ring.producer_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) if (flags & XDP_XMIT_FLUSH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) __tun_xdp_flush_tfile(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) return cnt - drops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) if (unlikely(!frame))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) return -EOVERFLOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) return tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) static const struct net_device_ops tap_netdev_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) .ndo_uninit = tun_net_uninit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) .ndo_open = tun_net_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) .ndo_stop = tun_net_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) .ndo_start_xmit = tun_net_xmit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) .ndo_fix_features = tun_net_fix_features,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) .ndo_set_rx_mode = tun_net_mclist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) .ndo_set_mac_address = eth_mac_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) .ndo_validate_addr = eth_validate_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) .ndo_select_queue = tun_select_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) .ndo_features_check = passthru_features_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) .ndo_set_rx_headroom = tun_set_headroom,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) .ndo_get_stats64 = tun_net_get_stats64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) .ndo_bpf = tun_xdp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) .ndo_xdp_xmit = tun_xdp_xmit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) .ndo_change_carrier = tun_net_change_carrier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) static void tun_flow_init(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) INIT_HLIST_HEAD(&tun->flows[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) tun->ageing_time = TUN_FLOW_EXPIRE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) mod_timer(&tun->flow_gc_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) round_jiffies_up(jiffies + tun->ageing_time));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) static void tun_flow_uninit(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) del_timer_sync(&tun->flow_gc_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) tun_flow_flush(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) #define MIN_MTU 68
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) #define MAX_MTU 65535
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) /* Initialize net device. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) static void tun_net_init(struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) switch (tun->flags & TUN_TYPE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) case IFF_TUN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) dev->netdev_ops = &tun_netdev_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) dev->header_ops = &ip_tunnel_header_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) /* Point-to-Point TUN Device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) dev->hard_header_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) dev->addr_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) dev->mtu = 1500;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) /* Zero header length */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) dev->type = ARPHRD_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) case IFF_TAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) dev->netdev_ops = &tap_netdev_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) /* Ethernet TAP Device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) ether_setup(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) dev->priv_flags &= ~IFF_TX_SKB_SHARING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) eth_hw_addr_random(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) dev->min_mtu = MIN_MTU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) dev->max_mtu = MAX_MTU - dev->hard_header_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) struct sock *sk = tfile->socket.sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) return (tun->dev->flags & IFF_UP) && sock_writeable(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) /* Character device part */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) /* Poll */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) static __poll_t tun_chr_poll(struct file *file, poll_table *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) struct tun_struct *tun = tun_get(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) struct sock *sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) __poll_t mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) if (!tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) return EPOLLERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) sk = tfile->socket.sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) poll_wait(file, sk_sleep(sk), wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) if (!ptr_ring_empty(&tfile->tx_ring))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) mask |= EPOLLIN | EPOLLRDNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) /* Make sure SOCKWQ_ASYNC_NOSPACE is set if not writable to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) * guarantee EPOLLOUT to be raised by either here or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) * tun_sock_write_space(). Then process could get notification
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) * after it writes to a down device and meets -EIO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) if (tun_sock_writeable(tun, tfile) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) tun_sock_writeable(tun, tfile)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) mask |= EPOLLOUT | EPOLLWRNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) if (tun->dev->reg_state != NETREG_REGISTERED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) mask = EPOLLERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) tun_put(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) return mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) const struct iov_iter *it)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) size_t linear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) if (it->nr_segs > MAX_SKB_FRAGS + 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) return ERR_PTR(-EMSGSIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) skb = napi_get_frags(&tfile->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) if (!skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) linear = iov_iter_single_seg_count(it);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) err = __skb_grow(skb, linear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) skb->len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) skb->data_len = len - linear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) skb->truesize += skb->data_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) for (i = 1; i < it->nr_segs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) size_t fragsz = it->iov[i].iov_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) void *frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) if (fragsz == 0 || fragsz > PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) frag = netdev_alloc_frag(fragsz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) if (!frag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) page = virt_to_head_page(frag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) skb_fill_page_desc(skb, i - 1, page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) frag - page_address(page), fragsz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) /* frees skb and all frags allocated with napi_alloc_frag() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) napi_free_frags(&tfile->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) return ERR_PTR(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) /* prepad is the amount to reserve at front. len is length after that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) * linear is a hint as to how much to copy (usually headers). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) static struct sk_buff *tun_alloc_skb(struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) size_t prepad, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) size_t linear, int noblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) struct sock *sk = tfile->socket.sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) /* Under a page? Don't bother with paged skb. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) if (prepad + len < PAGE_SIZE || !linear)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) linear = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) &err, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (!skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) return ERR_PTR(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) skb_reserve(skb, prepad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) skb_put(skb, linear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) skb->data_len = len - linear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) skb->len += len - linear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) struct sk_buff *skb, int more)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) struct sk_buff_head process_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) u32 rx_batched = tun->rx_batched;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) bool rcv = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) if (!rx_batched || (!more && skb_queue_empty(queue))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) skb_record_rx_queue(skb, tfile->queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) netif_receive_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) spin_lock(&queue->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) if (!more || skb_queue_len(queue) == rx_batched) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) __skb_queue_head_init(&process_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) skb_queue_splice_tail_init(queue, &process_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) rcv = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) __skb_queue_tail(queue, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) spin_unlock(&queue->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) if (rcv) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) struct sk_buff *nskb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) while ((nskb = __skb_dequeue(&process_queue))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) skb_record_rx_queue(nskb, tfile->queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) netif_receive_skb(nskb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) skb_record_rx_queue(skb, tfile->queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) netif_receive_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) int len, int noblock, bool zerocopy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) if (tfile->socket.sk->sk_sndbuf != INT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) if (!noblock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) if (zerocopy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) if (SKB_DATA_ALIGN(len + TUN_RX_PAD) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) static struct sk_buff *__tun_build_skb(struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) struct page_frag *alloc_frag, char *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) int buflen, int len, int pad)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) struct sk_buff *skb = build_skb(buf, buflen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) if (!skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) skb_reserve(skb, pad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) skb_put(skb, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) skb_set_owner_w(skb, tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) get_page(alloc_frag->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) alloc_frag->offset += buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) struct xdp_buff *xdp, u32 act)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) switch (act) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) case XDP_REDIRECT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) err = xdp_do_redirect(tun->dev, xdp, xdp_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) case XDP_TX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) err = tun_xdp_tx(tun->dev, xdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) case XDP_PASS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) bpf_warn_invalid_xdp_action(act);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) case XDP_ABORTED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) trace_xdp_exception(tun->dev, xdp_prog, act);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) case XDP_DROP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) this_cpu_inc(tun->pcpu_stats->rx_dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) return act;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) static struct sk_buff *tun_build_skb(struct tun_struct *tun,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) struct iov_iter *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) struct virtio_net_hdr *hdr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) int len, int *skb_xdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) struct page_frag *alloc_frag = ¤t->task_frag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) struct bpf_prog *xdp_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) char *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) size_t copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) int pad = TUN_RX_PAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) xdp_prog = rcu_dereference(tun->xdp_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) if (xdp_prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) pad += XDP_PACKET_HEADROOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) buflen += SKB_DATA_ALIGN(len + pad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) copied = copy_page_from_iter(alloc_frag->page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) alloc_frag->offset + pad,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) len, from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) if (copied != len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) return ERR_PTR(-EFAULT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) /* There's a small window that XDP may be set after the check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) * of xdp_prog above, this should be rare and for simplicity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) * we do XDP on skb in case the headroom is not enough.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) if (hdr->gso_type || !xdp_prog) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) *skb_xdp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) return __tun_build_skb(tfile, alloc_frag, buf, buflen, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) pad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) *skb_xdp = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) xdp_prog = rcu_dereference(tun->xdp_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) if (xdp_prog) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) struct xdp_buff xdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) u32 act;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) xdp.data_hard_start = buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) xdp.data = buf + pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) xdp_set_data_meta_invalid(&xdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) xdp.data_end = xdp.data + len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) xdp.rxq = &tfile->xdp_rxq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) xdp.frame_sz = buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) act = bpf_prog_run_xdp(xdp_prog, &xdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) if (act == XDP_REDIRECT || act == XDP_TX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) get_page(alloc_frag->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) alloc_frag->offset += buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) err = tun_xdp_act(tun, xdp_prog, &xdp, act);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) if (act == XDP_REDIRECT || act == XDP_TX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) put_page(alloc_frag->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) if (err == XDP_REDIRECT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) xdp_do_flush();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) if (err != XDP_PASS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) pad = xdp.data - xdp.data_hard_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) len = xdp.data_end - xdp.data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) /* Get packet from user space buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) void *msg_control, struct iov_iter *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) int noblock, bool more)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) size_t total_len = iov_iter_count(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) size_t len = total_len, align = tun->align, linear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) struct virtio_net_hdr gso = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) struct tun_pcpu_stats *stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) int good_linear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) int copylen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) bool zerocopy = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) u32 rxhash = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) int skb_xdp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) bool frags = tun_napi_frags_enabled(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) if (!(tun->flags & IFF_NO_PI)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) if (len < sizeof(pi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) len -= sizeof(pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) if (!copy_from_iter_full(&pi, sizeof(pi), from))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) if (tun->flags & IFF_VNET_HDR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) if (len < vnet_hdr_sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) len -= vnet_hdr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) if (!copy_from_iter_full(&gso, sizeof(gso), from))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) if ((gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso.hdr_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) if (tun16_to_cpu(tun, gso.hdr_len) > len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) iov_iter_advance(from, vnet_hdr_sz - sizeof(gso));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) align += NET_IP_ALIGN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) if (unlikely(len < ETH_HLEN ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) good_linear = SKB_MAX_HEAD(align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) if (msg_control) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) struct iov_iter i = *from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) /* There are 256 bytes to be copied in skb, so there is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) * enough room for skb expand head in case it is used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) * The rest of the buffer is mapped from userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) if (copylen > good_linear)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) copylen = good_linear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) linear = copylen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) iov_iter_advance(&i, copylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) zerocopy = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) /* For the packet that is not easy to be processed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) * (e.g gso or jumbo packet), we will do it at after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) * skb was created with generic XDP routine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) if (IS_ERR(skb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) this_cpu_inc(tun->pcpu_stats->rx_dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) return PTR_ERR(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) if (!skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) return total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) if (!zerocopy) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) copylen = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) if (tun16_to_cpu(tun, gso.hdr_len) > good_linear)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) linear = good_linear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) linear = tun16_to_cpu(tun, gso.hdr_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) if (frags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) mutex_lock(&tfile->napi_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) skb = tun_napi_alloc_frags(tfile, copylen, from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) /* tun_napi_alloc_frags() enforces a layout for the skb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) * If zerocopy is enabled, then this layout will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) * overwritten by zerocopy_sg_from_iter().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) zerocopy = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) skb = tun_alloc_skb(tfile, align, copylen, linear,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) noblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) if (IS_ERR(skb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) if (PTR_ERR(skb) != -EAGAIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) this_cpu_inc(tun->pcpu_stats->rx_dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) if (frags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) mutex_unlock(&tfile->napi_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) return PTR_ERR(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) if (zerocopy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) err = zerocopy_sg_from_iter(skb, from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) err = skb_copy_datagram_from_iter(skb, 0, from, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) err = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) drop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) this_cpu_inc(tun->pcpu_stats->rx_dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) if (frags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) tfile->napi.skb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) mutex_unlock(&tfile->napi_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) this_cpu_inc(tun->pcpu_stats->rx_frame_errors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) if (frags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) tfile->napi.skb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) mutex_unlock(&tfile->napi_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) switch (tun->flags & TUN_TYPE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) case IFF_TUN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) if (tun->flags & IFF_NO_PI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) switch (ip_version) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) case 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) pi.proto = htons(ETH_P_IP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) case 6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) pi.proto = htons(ETH_P_IPV6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) this_cpu_inc(tun->pcpu_stats->rx_dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) skb_reset_mac_header(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) skb->protocol = pi.proto;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) skb->dev = tun->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) case IFF_TAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) if (frags && !pskb_may_pull(skb, ETH_HLEN)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) skb->protocol = eth_type_trans(skb, tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) /* copy skb_ubuf_info for callback when skb has no error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) if (zerocopy) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) skb_shinfo(skb)->destructor_arg = msg_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) } else if (msg_control) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) struct ubuf_info *uarg = msg_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) uarg->callback(uarg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) skb_reset_network_header(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) skb_probe_transport_header(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) skb_record_rx_queue(skb, tfile->queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) if (skb_xdp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) struct bpf_prog *xdp_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) xdp_prog = rcu_dereference(tun->xdp_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) if (xdp_prog) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) ret = do_xdp_generic(xdp_prog, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) if (ret != XDP_PASS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) if (frags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) tfile->napi.skb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) mutex_unlock(&tfile->napi_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) return total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) /* Compute the costly rx hash only if needed for flow updates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) * We may get a very small possibility of OOO during switching, not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) * worth to optimize.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) !tfile->detached)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) rxhash = __skb_get_hash_symmetric(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) if (unlikely(!(tun->dev->flags & IFF_UP))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) if (frags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) u32 headlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) /* Exercise flow dissector code path. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) skb_push(skb, ETH_HLEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) headlen = eth_get_headlen(tun->dev, skb->data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) skb_headlen(skb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) if (unlikely(headlen > skb_headlen(skb))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) this_cpu_inc(tun->pcpu_stats->rx_dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) napi_free_frags(&tfile->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) mutex_unlock(&tfile->napi_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) napi_gro_frags(&tfile->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) mutex_unlock(&tfile->napi_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) } else if (tfile->napi_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) int queue_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) spin_lock_bh(&queue->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) __skb_queue_tail(queue, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) queue_len = skb_queue_len(queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) spin_unlock(&queue->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) if (!more || queue_len > NAPI_POLL_WEIGHT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) napi_schedule(&tfile->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) } else if (!IS_ENABLED(CONFIG_4KSTACKS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) tun_rx_batched(tun, tfile, skb, more);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) netif_rx_ni(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) stats = get_cpu_ptr(tun->pcpu_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) u64_stats_update_begin(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) u64_stats_inc(&stats->rx_packets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) u64_stats_add(&stats->rx_bytes, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) u64_stats_update_end(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) put_cpu_ptr(stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) if (rxhash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) tun_flow_update(tun, rxhash, tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) return total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) struct file *file = iocb->ki_filp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) struct tun_struct *tun = tun_get(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) ssize_t result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) int noblock = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) if (!tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) return -EBADFD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) noblock = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) result = tun_get_user(tun, tfile, NULL, from, noblock, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) tun_put(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) static ssize_t tun_put_user_xdp(struct tun_struct *tun,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) struct xdp_frame *xdp_frame,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) struct iov_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) int vnet_hdr_sz = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) size_t size = xdp_frame->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) struct tun_pcpu_stats *stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) size_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) if (tun->flags & IFF_VNET_HDR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) struct virtio_net_hdr gso = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) if (unlikely(iov_iter_count(iter) < vnet_hdr_sz))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) if (unlikely(copy_to_iter(&gso, sizeof(gso), iter) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) sizeof(gso)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) stats = get_cpu_ptr(tun->pcpu_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) u64_stats_update_begin(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) u64_stats_inc(&stats->tx_packets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) u64_stats_add(&stats->tx_bytes, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) u64_stats_update_end(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) put_cpu_ptr(tun->pcpu_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) /* Put packet to the user space buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) static ssize_t tun_put_user(struct tun_struct *tun,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) struct iov_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) struct tun_pi pi = { 0, skb->protocol };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) struct tun_pcpu_stats *stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) ssize_t total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) int vlan_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) int vlan_hlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) int vnet_hdr_sz = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) if (skb_vlan_tag_present(skb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) vlan_hlen = VLAN_HLEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) if (tun->flags & IFF_VNET_HDR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) total = skb->len + vlan_hlen + vnet_hdr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) if (!(tun->flags & IFF_NO_PI)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) if (iov_iter_count(iter) < sizeof(pi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) total += sizeof(pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) if (iov_iter_count(iter) < total) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) /* Packet will be striped */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) pi.flags |= TUN_PKT_STRIP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) if (vnet_hdr_sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) struct virtio_net_hdr gso;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) if (iov_iter_count(iter) < vnet_hdr_sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) if (virtio_net_hdr_from_skb(skb, &gso,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) tun_is_little_endian(tun), true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) vlan_hlen)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) struct skb_shared_info *sinfo = skb_shinfo(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) pr_err("unexpected GSO type: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) "0x%x, gso_size %d, hdr_len %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) tun16_to_cpu(tun, gso.hdr_len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) print_hex_dump(KERN_ERR, "tun: ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) DUMP_PREFIX_NONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) 16, 1, skb->head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) if (copy_to_iter(&gso, sizeof(gso), iter) != sizeof(gso))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) if (vlan_hlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) struct veth veth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) veth.h_vlan_proto = skb->vlan_proto;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) if (ret || !iov_iter_count(iter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) ret = copy_to_iter(&veth, sizeof(veth), iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) if (ret != sizeof(veth) || !iov_iter_count(iter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) /* caller is in process context, */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) stats = get_cpu_ptr(tun->pcpu_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) u64_stats_update_begin(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) u64_stats_inc(&stats->tx_packets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) u64_stats_add(&stats->tx_bytes, skb->len + vlan_hlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) u64_stats_update_end(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) put_cpu_ptr(tun->pcpu_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) return total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) DECLARE_WAITQUEUE(wait, current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) void *ptr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) int error = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) ptr = ptr_ring_consume(&tfile->tx_ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) if (ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) if (noblock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) error = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) add_wait_queue(&tfile->socket.wq.wait, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) set_current_state(TASK_INTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) ptr = ptr_ring_consume(&tfile->tx_ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) if (ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) if (signal_pending(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) error = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) error = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) __set_current_state(TASK_RUNNING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) remove_wait_queue(&tfile->socket.wq.wait, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) *err = error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) return ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) struct iov_iter *to,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) int noblock, void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) ssize_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) if (!iov_iter_count(to)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) tun_ptr_free(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) if (!ptr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) /* Read frames from ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) ptr = tun_ring_recv(tfile, noblock, &err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) if (!ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) if (tun_is_xdp_frame(ptr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) ret = tun_put_user_xdp(tun, tfile, xdpf, to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) xdp_return_frame(xdpf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) struct sk_buff *skb = ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) ret = tun_put_user(tun, tfile, skb, to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) if (unlikely(ret < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) consume_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) struct file *file = iocb->ki_filp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) struct tun_struct *tun = tun_get(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) ssize_t len = iov_iter_count(to), ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) int noblock = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) if (!tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) return -EBADFD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) noblock = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) ret = tun_do_read(tun, tfile, to, noblock, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) ret = min_t(ssize_t, ret, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) iocb->ki_pos = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) tun_put(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) static void tun_prog_free(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) bpf_prog_destroy(prog->prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) kfree(prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) static int __tun_set_ebpf(struct tun_struct *tun,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) struct tun_prog __rcu **prog_p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) struct bpf_prog *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) struct tun_prog *old, *new = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) if (prog) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) new = kmalloc(sizeof(*new), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) new->prog = prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) spin_lock_bh(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) old = rcu_dereference_protected(*prog_p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) lockdep_is_held(&tun->lock));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) rcu_assign_pointer(*prog_p, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) spin_unlock_bh(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) if (old)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) call_rcu(&old->rcu, tun_prog_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) static void tun_free_netdev(struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) BUG_ON(!(list_empty(&tun->disabled)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) free_percpu(tun->pcpu_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) /* We clear pcpu_stats so that tun_set_iff() can tell if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) * tun_free_netdev() has been called from register_netdevice().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) tun->pcpu_stats = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) tun_flow_uninit(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) security_tun_dev_free_security(tun->security);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) __tun_set_ebpf(tun, &tun->steering_prog, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) __tun_set_ebpf(tun, &tun->filter_prog, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) static void tun_setup(struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) tun->owner = INVALID_UID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) tun->group = INVALID_GID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) tun_default_link_ksettings(dev, &tun->link_ksettings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) dev->ethtool_ops = &tun_ethtool_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) dev->needs_free_netdev = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) dev->priv_destructor = tun_free_netdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) /* We prefer our own queue length */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) dev->tx_queue_len = TUN_READQ_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) /* Trivial set of netlink ops to allow deleting tun or tap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) * device with netlink.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) static int tun_validate(struct nlattr *tb[], struct nlattr *data[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) NL_SET_ERR_MSG(extack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) "tun/tap creation via rtnetlink is not supported.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) static size_t tun_get_size(const struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) return nla_total_size(sizeof(uid_t)) + /* OWNER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) nla_total_size(sizeof(gid_t)) + /* GROUP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) nla_total_size(sizeof(u8)) + /* TYPE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) nla_total_size(sizeof(u8)) + /* PI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) nla_total_size(sizeof(u8)) + /* VNET_HDR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) nla_total_size(sizeof(u8)) + /* PERSIST */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) nla_total_size(sizeof(u8)) + /* MULTI_QUEUE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) nla_total_size(sizeof(u32)) + /* NUM_QUEUES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) nla_total_size(sizeof(u32)) + /* NUM_DISABLED_QUEUES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) if (uid_valid(tun->owner) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) nla_put_u32(skb, IFLA_TUN_OWNER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) from_kuid_munged(current_user_ns(), tun->owner)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) if (gid_valid(tun->group) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) nla_put_u32(skb, IFLA_TUN_GROUP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) from_kgid_munged(current_user_ns(), tun->group)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) !!(tun->flags & IFF_MULTI_QUEUE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) if (tun->flags & IFF_MULTI_QUEUE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) tun->numdisabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) nla_put_failure:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) return -EMSGSIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) static struct rtnl_link_ops tun_link_ops __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) .kind = DRV_NAME,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) .priv_size = sizeof(struct tun_struct),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) .setup = tun_setup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) .validate = tun_validate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) .get_size = tun_get_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) .fill_info = tun_fill_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) static void tun_sock_write_space(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) wait_queue_head_t *wqueue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) if (!sock_writeable(sk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) wqueue = sk_sleep(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) if (wqueue && waitqueue_active(wqueue))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) wake_up_interruptible_sync_poll(wqueue, EPOLLOUT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) EPOLLWRNORM | EPOLLWRBAND);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) tfile = container_of(sk, struct tun_file, sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) static void tun_put_page(struct tun_page *tpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) if (tpage->page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) __page_frag_cache_drain(tpage->page, tpage->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) static int tun_xdp_one(struct tun_struct *tun,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) struct tun_file *tfile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) struct xdp_buff *xdp, int *flush,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) struct tun_page *tpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) unsigned int datasize = xdp->data_end - xdp->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) struct tun_xdp_hdr *hdr = xdp->data_hard_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) struct virtio_net_hdr *gso = &hdr->gso;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) struct tun_pcpu_stats *stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) struct bpf_prog *xdp_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) struct sk_buff *skb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) u32 rxhash = 0, act;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) int buflen = hdr->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) bool skb_xdp = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) xdp_prog = rcu_dereference(tun->xdp_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) if (xdp_prog) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) if (gso->gso_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) skb_xdp = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) goto build;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) xdp_set_data_meta_invalid(xdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) xdp->rxq = &tfile->xdp_rxq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) xdp->frame_sz = buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) act = bpf_prog_run_xdp(xdp_prog, xdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) err = tun_xdp_act(tun, xdp_prog, xdp, act);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) put_page(virt_to_head_page(xdp->data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) switch (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) case XDP_REDIRECT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) *flush = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) case XDP_TX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) case XDP_PASS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) page = virt_to_head_page(xdp->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) if (tpage->page == page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) ++tpage->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) tun_put_page(tpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) tpage->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) tpage->count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) build:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) skb = build_skb(xdp->data_hard_start, buflen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) if (!skb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) skb_reserve(skb, xdp->data - xdp->data_hard_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) skb_put(skb, xdp->data_end - xdp->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) if (virtio_net_hdr_to_skb(skb, gso, tun_is_little_endian(tun))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) this_cpu_inc(tun->pcpu_stats->rx_frame_errors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) skb->protocol = eth_type_trans(skb, tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) skb_reset_network_header(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) skb_probe_transport_header(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) skb_record_rx_queue(skb, tfile->queue_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) if (skb_xdp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) err = do_xdp_generic(xdp_prog, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) if (err != XDP_PASS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) !tfile->detached)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) rxhash = __skb_get_hash_symmetric(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) netif_receive_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) /* No need for get_cpu_ptr() here since this function is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) * always called with bh disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) stats = this_cpu_ptr(tun->pcpu_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) u64_stats_update_begin(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) u64_stats_inc(&stats->rx_packets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) u64_stats_add(&stats->rx_bytes, datasize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) u64_stats_update_end(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) if (rxhash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) tun_flow_update(tun, rxhash, tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) int ret, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) struct tun_file *tfile = container_of(sock, struct tun_file, socket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) struct tun_struct *tun = tun_get(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) struct tun_msg_ctl *ctl = m->msg_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) struct xdp_buff *xdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) if (!tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) return -EBADFD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) if (ctl && (ctl->type == TUN_MSG_PTR)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) struct tun_page tpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) int n = ctl->num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) int flush = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) memset(&tpage, 0, sizeof(tpage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) for (i = 0; i < n; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) xdp = &((struct xdp_buff *)ctl->ptr)[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) tun_xdp_one(tun, tfile, xdp, &flush, &tpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) if (flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) xdp_do_flush();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) tun_put_page(&tpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) ret = total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) m->msg_flags & MSG_DONTWAIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) m->msg_flags & MSG_MORE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) tun_put(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) struct tun_file *tfile = container_of(sock, struct tun_file, socket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) struct tun_struct *tun = tun_get(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) void *ptr = m->msg_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) if (!tun) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) ret = -EBADFD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) goto out_put_tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) if (flags & MSG_ERRQUEUE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) ret = sock_recv_errqueue(sock->sk, m, total_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) SOL_PACKET, TUN_TX_TIMESTAMP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) if (ret > (ssize_t)total_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) m->msg_flags |= MSG_TRUNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) ret = flags & MSG_TRUNC ? ret : total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) tun_put(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) out_put_tun:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) tun_put(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) tun_ptr_free(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) static int tun_ptr_peek_len(void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) if (likely(ptr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) if (tun_is_xdp_frame(ptr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) return xdpf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) return __skb_array_len_with_tag(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) static int tun_peek_len(struct socket *sock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) struct tun_file *tfile = container_of(sock, struct tun_file, socket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) struct tun_struct *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) tun = tun_get(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) if (!tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) tun_put(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) /* Ops structure to mimic raw sockets with tun */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) static const struct proto_ops tun_socket_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) .peek_len = tun_peek_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) .sendmsg = tun_sendmsg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) .recvmsg = tun_recvmsg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) static struct proto tun_proto = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) .name = "tun",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) .obj_size = sizeof(struct tun_file),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) static int tun_flags(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) static ssize_t tun_show_flags(struct device *dev, struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) struct tun_struct *tun = netdev_priv(to_net_dev(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) return sprintf(buf, "0x%x\n", tun_flags(tun));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) static ssize_t tun_show_owner(struct device *dev, struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) struct tun_struct *tun = netdev_priv(to_net_dev(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) return uid_valid(tun->owner)?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) sprintf(buf, "%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) from_kuid_munged(current_user_ns(), tun->owner)):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) sprintf(buf, "-1\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) static ssize_t tun_show_group(struct device *dev, struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) struct tun_struct *tun = netdev_priv(to_net_dev(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) return gid_valid(tun->group) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) sprintf(buf, "%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) from_kgid_munged(current_user_ns(), tun->group)):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) sprintf(buf, "-1\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) static DEVICE_ATTR(tun_flags, 0444, tun_show_flags, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) static DEVICE_ATTR(owner, 0444, tun_show_owner, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) static DEVICE_ATTR(group, 0444, tun_show_group, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) static struct attribute *tun_dev_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) &dev_attr_tun_flags.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) &dev_attr_owner.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) &dev_attr_group.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) static const struct attribute_group tun_attr_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) .attrs = tun_dev_attrs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) struct tun_struct *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) struct net_device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) if (tfile->detached)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) if (!capable(CAP_NET_ADMIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) if (!(ifr->ifr_flags & IFF_NAPI) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) dev = __dev_get_by_name(net, ifr->ifr_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) if (dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) if (ifr->ifr_flags & IFF_TUN_EXCL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) !!(tun->flags & IFF_MULTI_QUEUE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) if (tun_not_capable(tun))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) err = security_tun_dev_open(tun->security);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) ifr->ifr_flags & IFF_NAPI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) ifr->ifr_flags & IFF_NAPI_FRAGS, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) if (tun->flags & IFF_MULTI_QUEUE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) (tun->numqueues + tun->numdisabled > 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) /* One or more queue has already been attached, no need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) * to initialize the device again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) netdev_state_change(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) tun->flags = (tun->flags & ~TUN_FEATURES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) (ifr->ifr_flags & TUN_FEATURES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) netdev_state_change(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) unsigned long flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) MAX_TAP_QUEUES : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) err = security_tun_dev_create();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) /* Set dev type */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) if (ifr->ifr_flags & IFF_TUN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) /* TUN device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) flags |= IFF_TUN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) name = "tun%d";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) } else if (ifr->ifr_flags & IFF_TAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) /* TAP device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) flags |= IFF_TAP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) name = "tap%d";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) if (*ifr->ifr_name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) name = ifr->ifr_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) dev = alloc_netdev_mqs(sizeof(struct tun_struct), name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) NET_NAME_UNKNOWN, tun_setup, queues,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) queues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) if (!dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) dev_net_set(dev, net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) dev->rtnl_link_ops = &tun_link_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) dev->ifindex = tfile->ifindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) dev->sysfs_groups[0] = &tun_attr_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) tun->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) tun->flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) tun->txflt.count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) tun->align = NET_SKB_PAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) tun->filter_attached = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) tun->sndbuf = tfile->socket.sk->sk_sndbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) tun->rx_batched = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) RCU_INIT_POINTER(tun->steering_prog, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) tun->pcpu_stats = netdev_alloc_pcpu_stats(struct tun_pcpu_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) if (!tun->pcpu_stats) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) goto err_free_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) spin_lock_init(&tun->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) err = security_tun_dev_alloc_security(&tun->security);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) goto err_free_stat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) tun_net_init(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) tun_flow_init(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) NETIF_F_HW_VLAN_STAG_TX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) dev->features = dev->hw_features | NETIF_F_LLTX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) dev->vlan_features = dev->features &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) ~(NETIF_F_HW_VLAN_CTAG_TX |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) NETIF_F_HW_VLAN_STAG_TX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) tun->flags = (tun->flags & ~TUN_FEATURES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) (ifr->ifr_flags & TUN_FEATURES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) INIT_LIST_HEAD(&tun->disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) err = tun_attach(tun, file, false, ifr->ifr_flags & IFF_NAPI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) ifr->ifr_flags & IFF_NAPI_FRAGS, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) goto err_free_flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) err = register_netdevice(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) goto err_detach;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) /* free_netdev() won't check refcnt, to aovid race
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) * with dev_put() we need publish tun after registration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) rcu_assign_pointer(tfile->tun, tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) netif_carrier_on(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) /* Make sure persistent devices do not get stuck in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) * xoff state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) if (netif_running(tun->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) netif_tx_wake_all_queues(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) strcpy(ifr->ifr_name, tun->dev->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) err_detach:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) tun_detach_all(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) /* We are here because register_netdevice() has failed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) * If register_netdevice() already called tun_free_netdev()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) * while dealing with the error, tun->pcpu_stats has been cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) if (!tun->pcpu_stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) goto err_free_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) err_free_flow:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) tun_flow_uninit(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) security_tun_dev_free_security(tun->security);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) err_free_stat:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) free_percpu(tun->pcpu_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) err_free_dev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) free_netdev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) strcpy(ifr->ifr_name, tun->dev->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) ifr->ifr_flags = tun_flags(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) /* This is like a cut-down ethtool ops, except done via tun fd so no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) * privs required. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) static int set_offload(struct tun_struct *tun, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) netdev_features_t features = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) if (arg & TUN_F_CSUM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) features |= NETIF_F_HW_CSUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) arg &= ~TUN_F_CSUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) if (arg & (TUN_F_TSO4|TUN_F_TSO6)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) if (arg & TUN_F_TSO_ECN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) features |= NETIF_F_TSO_ECN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) arg &= ~TUN_F_TSO_ECN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) if (arg & TUN_F_TSO4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) features |= NETIF_F_TSO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) if (arg & TUN_F_TSO6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) features |= NETIF_F_TSO6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) arg &= ~(TUN_F_TSO4|TUN_F_TSO6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) arg &= ~TUN_F_UFO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) /* This gives the user a way to test for new features in future by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) * trying to set them. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) if (arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) tun->set_features = features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) tun->dev->wanted_features &= ~TUN_USER_FEATURES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) tun->dev->wanted_features |= features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) netdev_update_features(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) static void tun_detach_filter(struct tun_struct *tun, int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) for (i = 0; i < n; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) tfile = rtnl_dereference(tun->tfiles[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) lock_sock(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) sk_detach_filter(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) release_sock(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) tun->filter_attached = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) static int tun_attach_filter(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) int i, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) for (i = 0; i < tun->numqueues; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) tfile = rtnl_dereference(tun->tfiles[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) lock_sock(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) ret = sk_attach_filter(&tun->fprog, tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) release_sock(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) tun_detach_filter(tun, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) tun->filter_attached = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) static void tun_set_sndbuf(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) for (i = 0; i < tun->numqueues; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) tfile = rtnl_dereference(tun->tfiles[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) tfile->socket.sk->sk_sndbuf = tun->sndbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) static int tun_set_queue(struct file *file, struct ifreq *ifr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) struct tun_struct *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) rtnl_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) if (ifr->ifr_flags & IFF_ATTACH_QUEUE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) tun = tfile->detached;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) if (!tun) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) ret = security_tun_dev_attach_queue(tun->security);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) tun->flags & IFF_NAPI_FRAGS, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) tun = rtnl_dereference(tfile->tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) __tun_detach(tfile, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) if (ret >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) netdev_state_change(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) rtnl_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog __rcu **prog_p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) void __user *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) struct bpf_prog *prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) int fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) if (copy_from_user(&fd, data, sizeof(fd)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) if (fd == -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) prog = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) if (IS_ERR(prog))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) return PTR_ERR(prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) return __tun_set_ebpf(tun, prog_p, prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) /* Return correct value for tun->dev->addr_len based on tun->dev->type. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) static unsigned char tun_get_addr_len(unsigned short type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) case ARPHRD_IP6GRE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) case ARPHRD_TUNNEL6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) return sizeof(struct in6_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) case ARPHRD_IPGRE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) case ARPHRD_TUNNEL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) case ARPHRD_SIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) return 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) case ARPHRD_ETHER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) return ETH_ALEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) case ARPHRD_IEEE802154:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) case ARPHRD_IEEE802154_MONITOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) return IEEE802154_EXTENDED_ADDR_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) case ARPHRD_PHONET_PIPE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) case ARPHRD_PPP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) case ARPHRD_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) case ARPHRD_6LOWPAN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) return EUI64_ADDR_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) case ARPHRD_FDDI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) return FDDI_K_ALEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) case ARPHRD_HIPPI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) return HIPPI_ALEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) case ARPHRD_IEEE802:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) return FC_ALEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) case ARPHRD_ROSE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) return ROSE_ADDR_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) case ARPHRD_NETROM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) return AX25_ADDR_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) case ARPHRD_LOCALTLK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) return LTALK_ALEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) unsigned long arg, int ifreq_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) struct net *net = sock_net(&tfile->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) struct tun_struct *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) void __user* argp = (void __user*)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) unsigned int ifindex, carrier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) struct ifreq ifr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) kuid_t owner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) kgid_t group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) int sndbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) int vnet_hdr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) int le;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) bool do_notify = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) if (cmd == TUNSETIFF || cmd == TUNSETQUEUE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) if (copy_from_user(&ifr, argp, ifreq_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) memset(&ifr, 0, sizeof(ifr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) if (cmd == TUNGETFEATURES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) /* Currently this just means: "what IFF flags are valid?".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) * This is needed because we never checked for invalid flags on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) * TUNSETIFF.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) return put_user(IFF_TUN | IFF_TAP | TUN_FEATURES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) (unsigned int __user*)argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) } else if (cmd == TUNSETQUEUE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) return tun_set_queue(file, &ifr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) } else if (cmd == SIOCGSKNS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) return open_related_ns(&net->ns, get_net_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) rtnl_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) tun = tun_get(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) if (cmd == TUNSETIFF) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) ret = -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) if (tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) ifr.ifr_name[IFNAMSIZ-1] = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) ret = tun_set_iff(net, file, &ifr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) if (copy_to_user(argp, &ifr, ifreq_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) if (cmd == TUNSETIFINDEX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) ret = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) if (tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) if (copy_from_user(&ifindex, argp, sizeof(ifindex)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) tfile->ifindex = ifindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) ret = -EBADFD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) if (!tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) netif_info(tun, drv, tun->dev, "tun_chr_ioctl cmd %u\n", cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) net = dev_net(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) case TUNGETIFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) tun_get_iff(tun, &ifr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) if (tfile->detached)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) ifr.ifr_flags |= IFF_DETACH_QUEUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) if (!tfile->socket.sk->sk_filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) ifr.ifr_flags |= IFF_NOFILTER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) if (copy_to_user(argp, &ifr, ifreq_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) case TUNSETNOCSUM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) /* Disable/Enable checksum */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) /* [unimplemented] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) netif_info(tun, drv, tun->dev, "ignored: set checksum %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) arg ? "disabled" : "enabled");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) case TUNSETPERSIST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) /* Disable/Enable persist mode. Keep an extra reference to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) * module to prevent the module being unprobed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) if (arg && !(tun->flags & IFF_PERSIST)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) tun->flags |= IFF_PERSIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) __module_get(THIS_MODULE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) do_notify = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) if (!arg && (tun->flags & IFF_PERSIST)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) tun->flags &= ~IFF_PERSIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) module_put(THIS_MODULE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) do_notify = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) netif_info(tun, drv, tun->dev, "persist %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) arg ? "enabled" : "disabled");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) case TUNSETOWNER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) /* Set owner of the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) owner = make_kuid(current_user_ns(), arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) if (!uid_valid(owner)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) tun->owner = owner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) do_notify = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) netif_info(tun, drv, tun->dev, "owner set to %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) from_kuid(&init_user_ns, tun->owner));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) case TUNSETGROUP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) /* Set group of the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) group = make_kgid(current_user_ns(), arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) if (!gid_valid(group)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) tun->group = group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) do_notify = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) netif_info(tun, drv, tun->dev, "group set to %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) from_kgid(&init_user_ns, tun->group));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) case TUNSETLINK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) /* Only allow setting the type when the interface is down */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) if (tun->dev->flags & IFF_UP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) netif_info(tun, drv, tun->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) "Linktype set failed because interface is up\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) tun->dev->type = (int) arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) tun->dev->addr_len = tun_get_addr_len(tun->dev->type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) netif_info(tun, drv, tun->dev, "linktype set to %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) tun->dev->type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) case TUNSETDEBUG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) tun->msg_enable = (u32)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) case TUNSETOFFLOAD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) ret = set_offload(tun, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) case TUNSETTXFILTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) /* Can be set only for TAPs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) ret = update_filter(&tun->txflt, (void __user *)arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) case SIOCGIFHWADDR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) /* Get hw address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) dev_get_mac_address(&ifr.ifr_hwaddr, net, tun->dev->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) if (copy_to_user(argp, &ifr, ifreq_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) case SIOCSIFHWADDR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) /* Set hw address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) ret = dev_set_mac_address_user(tun->dev, &ifr.ifr_hwaddr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) case TUNGETSNDBUF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) sndbuf = tfile->socket.sk->sk_sndbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) if (copy_to_user(argp, &sndbuf, sizeof(sndbuf)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) case TUNSETSNDBUF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) if (sndbuf <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) tun->sndbuf = sndbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) tun_set_sndbuf(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) case TUNGETVNETHDRSZ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) vnet_hdr_sz = tun->vnet_hdr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) if (copy_to_user(argp, &vnet_hdr_sz, sizeof(vnet_hdr_sz)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) case TUNSETVNETHDRSZ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) if (copy_from_user(&vnet_hdr_sz, argp, sizeof(vnet_hdr_sz))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) if (vnet_hdr_sz < (int)sizeof(struct virtio_net_hdr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) tun->vnet_hdr_sz = vnet_hdr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) case TUNGETVNETLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) le = !!(tun->flags & TUN_VNET_LE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) if (put_user(le, (int __user *)argp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) case TUNSETVNETLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) if (get_user(le, (int __user *)argp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) if (le)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) tun->flags |= TUN_VNET_LE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) tun->flags &= ~TUN_VNET_LE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) case TUNGETVNETBE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) ret = tun_get_vnet_be(tun, argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) case TUNSETVNETBE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) ret = tun_set_vnet_be(tun, argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) case TUNATTACHFILTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) /* Can be set only for TAPs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) ret = tun_attach_filter(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) case TUNDETACHFILTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) /* Can be set only for TAPs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) tun_detach_filter(tun, tun->numqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) case TUNGETFILTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) case TUNSETSTEERINGEBPF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) ret = tun_set_ebpf(tun, &tun->steering_prog, argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) case TUNSETFILTEREBPF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) ret = tun_set_ebpf(tun, &tun->filter_prog, argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) case TUNSETCARRIER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) if (copy_from_user(&carrier, argp, sizeof(carrier)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) ret = tun_net_change_carrier(tun->dev, (bool)carrier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) case TUNGETDEVNETNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) ret = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) ret = open_related_ns(&net->ns, get_net_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) if (do_notify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) netdev_state_change(tun->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) rtnl_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) if (tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) tun_put(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) static long tun_chr_ioctl(struct file *file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) unsigned int cmd, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) static long tun_chr_compat_ioctl(struct file *file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) unsigned int cmd, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) case TUNSETIFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) case TUNGETIFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) case TUNSETTXFILTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) case TUNGETSNDBUF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) case TUNSETSNDBUF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) case SIOCGIFHWADDR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) case SIOCSIFHWADDR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) arg = (unsigned long)compat_ptr(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) arg = (compat_ulong_t)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) * compat_ifreq is shorter than ifreq, so we must not access beyond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) * the end of that structure. All fields that are used in this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) * driver are compatible though, we don't need to convert the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) * contents.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) #endif /* CONFIG_COMPAT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) static int tun_chr_fasync(int fd, struct file *file, int on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) if (on) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) __f_setown(file, task_pid(current), PIDTYPE_TGID, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) tfile->flags |= TUN_FASYNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) tfile->flags &= ~TUN_FASYNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) static int tun_chr_open(struct inode *inode, struct file * file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) struct net *net = current->nsproxy->net_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) &tun_proto, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) if (!tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) sk_free(&tfile->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) mutex_init(&tfile->napi_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) RCU_INIT_POINTER(tfile->tun, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) tfile->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) tfile->ifindex = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) init_waitqueue_head(&tfile->socket.wq.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) tfile->socket.file = file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) tfile->socket.ops = &tun_socket_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) sock_init_data(&tfile->socket, &tfile->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) tfile->sk.sk_write_space = tun_sock_write_space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) tfile->sk.sk_sndbuf = INT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) file->private_data = tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) INIT_LIST_HEAD(&tfile->next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) static int tun_chr_close(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) tun_detach(tfile, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) #ifdef CONFIG_PROC_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) struct tun_file *tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) struct tun_struct *tun;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) struct ifreq ifr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) memset(&ifr, 0, sizeof(ifr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) rtnl_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465) tun = tun_get(tfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) if (tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) tun_get_iff(tun, &ifr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) rtnl_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) if (tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) tun_put(tun);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) seq_printf(m, "iff:\t%s\n", ifr.ifr_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) static const struct file_operations tun_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) .llseek = no_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) .read_iter = tun_chr_read_iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) .write_iter = tun_chr_write_iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) .poll = tun_chr_poll,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) .unlocked_ioctl = tun_chr_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) .compat_ioctl = tun_chr_compat_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) .open = tun_chr_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) .release = tun_chr_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) .fasync = tun_chr_fasync,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) #ifdef CONFIG_PROC_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) .show_fdinfo = tun_chr_show_fdinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) static struct miscdevice tun_miscdev = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) .minor = TUN_MINOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) .name = "tun",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) .nodename = "net/tun",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) .fops = &tun_fops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) /* ethtool interface */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) static void tun_default_link_ksettings(struct net_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) struct ethtool_link_ksettings *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) ethtool_link_ksettings_zero_link_mode(cmd, supported);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) ethtool_link_ksettings_zero_link_mode(cmd, advertising);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) cmd->base.speed = SPEED_10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) cmd->base.duplex = DUPLEX_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) cmd->base.port = PORT_TP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) cmd->base.phy_address = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) cmd->base.autoneg = AUTONEG_DISABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) static int tun_get_link_ksettings(struct net_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) struct ethtool_link_ksettings *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) memcpy(cmd, &tun->link_ksettings, sizeof(*cmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525) static int tun_set_link_ksettings(struct net_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) const struct ethtool_link_ksettings *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) memcpy(&tun->link_ksettings, cmd, sizeof(*cmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) strlcpy(info->version, DRV_VERSION, sizeof(info->version));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) switch (tun->flags & TUN_TYPE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) case IFF_TUN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) strlcpy(info->bus_info, "tun", sizeof(info->bus_info));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) case IFF_TAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) strlcpy(info->bus_info, "tap", sizeof(info->bus_info));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551) static u32 tun_get_msglevel(struct net_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) return tun->msg_enable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) static void tun_set_msglevel(struct net_device *dev, u32 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) tun->msg_enable = value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) static int tun_get_coalesce(struct net_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) struct ethtool_coalesce *ec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570) ec->rx_max_coalesced_frames = tun->rx_batched;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) static int tun_set_coalesce(struct net_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) struct ethtool_coalesce *ec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) tun->rx_batched = NAPI_POLL_WEIGHT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) tun->rx_batched = ec->rx_max_coalesced_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) static const struct ethtool_ops tun_ethtool_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) .supported_coalesce_params = ETHTOOL_COALESCE_RX_MAX_FRAMES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) .get_drvinfo = tun_get_drvinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) .get_msglevel = tun_get_msglevel,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) .set_msglevel = tun_set_msglevel,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593) .get_link = ethtool_op_get_link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) .get_ts_info = ethtool_op_get_ts_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) .get_coalesce = tun_get_coalesce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596) .set_coalesce = tun_set_coalesce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) .get_link_ksettings = tun_get_link_ksettings,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598) .set_link_ksettings = tun_set_link_ksettings,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) static int tun_queue_resize(struct tun_struct *tun)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) struct net_device *dev = tun->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) struct ptr_ring **rings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) int n = tun->numqueues + tun->numdisabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) int ret, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) if (!rings)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) for (i = 0; i < tun->numqueues; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) tfile = rtnl_dereference(tun->tfiles[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) rings[i] = &tfile->tx_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) list_for_each_entry(tfile, &tun->disabled, next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618) rings[i++] = &tfile->tx_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) ret = ptr_ring_resize_multiple(rings, n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) dev->tx_queue_len, GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) tun_ptr_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) kfree(rings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) static int tun_device_event(struct notifier_block *unused,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) unsigned long event, void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) struct net_device *dev = netdev_notifier_info_to_dev(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) struct tun_struct *tun = netdev_priv(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) if (dev->rtnl_link_ops != &tun_link_ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) return NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) switch (event) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) case NETDEV_CHANGE_TX_QUEUE_LEN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640) if (tun_queue_resize(tun))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) return NOTIFY_BAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) case NETDEV_UP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) for (i = 0; i < tun->numqueues; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647) tfile = rtnl_dereference(tun->tfiles[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) tfile->socket.sk->sk_write_space(tfile->socket.sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) return NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) static struct notifier_block tun_notifier_block __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) .notifier_call = tun_device_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) static int __init tun_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) ret = rtnl_link_register(&tun_link_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) pr_err("Can't register link_ops\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) goto err_linkops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) ret = misc_register(&tun_miscdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) pr_err("Can't register misc device %d\n", TUN_MINOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) goto err_misc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) ret = register_netdevice_notifier(&tun_notifier_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) pr_err("Can't register netdevice notifier\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) goto err_notifier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) err_notifier:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) misc_deregister(&tun_miscdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) err_misc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) rtnl_link_unregister(&tun_link_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) err_linkops:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) static void tun_cleanup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) misc_deregister(&tun_miscdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699) rtnl_link_unregister(&tun_link_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) unregister_netdevice_notifier(&tun_notifier_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) /* Get an underlying socket object from tun file. Returns error unless file is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) * attached to a device. The returned object works like a packet socket, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) * holding a reference to the file for as long as the socket is in use. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) struct socket *tun_get_socket(struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) if (file->f_op != &tun_fops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) if (!tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714) return ERR_PTR(-EBADFD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) return &tfile->socket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717) EXPORT_SYMBOL_GPL(tun_get_socket);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) struct ptr_ring *tun_get_tx_ring(struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) struct tun_file *tfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) if (file->f_op != &tun_fops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725) tfile = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) if (!tfile)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727) return ERR_PTR(-EBADFD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) return &tfile->tx_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) EXPORT_SYMBOL_GPL(tun_get_tx_ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) module_init(tun_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) module_exit(tun_cleanup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) MODULE_DESCRIPTION(DRV_DESCRIPTION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735) MODULE_AUTHOR(DRV_COPYRIGHT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737) MODULE_ALIAS_MISCDEV(TUN_MINOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) MODULE_ALIAS("devname:net/tun");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags