/* * Virtual network driver for conversing with remote driver backends. * * Copyright (c) 2002-2005, K A Fraser * Copyright (c) 2005, XenSource Ltd * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation; or, when distributed * separately from the Linux kernel or incorporated into other * software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/module.h> #include <linux/kernel.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/skbuff.h> #include <linux/ethtool.h> #include <linux/if_ether.h> #include <net/tcp.h> #include <linux/udp.h> #include <linux/moduleparam.h> #include <linux/mm.h> #include <linux/slab.h> #include <net/ip.h> #include <linux/bpf.h> #include <net/page_pool.h> #include <linux/bpf_trace.h> #include <xen/xen.h> #include <xen/xenbus.h> #include <xen/events.h> #include <xen/page.h> #include <xen/platform_pci.h> #include <xen/grant_table.h> #include <xen/interface/io/netif.h> #include <xen/interface/memory.h> #include <xen/interface/grant_table.h> /* Module parameters */ #define MAX_QUEUES_DEFAULT 8 static unsigned int xennet_max_queues; module_param_named(max_queues, xennet_max_queues, uint, 0644); MODULE_PARM_DESC(max_queues, <------><------> "Maximum number of queues per virtual interface"); #define XENNET_TIMEOUT (5 * HZ) static const struct ethtool_ops xennet_ethtool_ops; struct netfront_cb { <------>int pull_to; }; #define NETFRONT_SKB_CB(skb) ((struct netfront_cb *)((skb)->cb)) #define RX_COPY_THRESHOLD 256 #define GRANT_INVALID_REF 0 #define NET_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE) #define NET_RX_RING_SIZE __CONST_RING_SIZE(xen_netif_rx, XEN_PAGE_SIZE) /* Minimum number of Rx slots (includes slot for GSO metadata). */ #define NET_RX_SLOTS_MIN (XEN_NETIF_NR_SLOTS_MIN + 1) /* Queue name is interface name with "-qNNN" appended */ #define QUEUE_NAME_SIZE (IFNAMSIZ + 6) /* IRQ name is queue name with "-tx" or "-rx" appended */ #define IRQ_NAME_SIZE (QUEUE_NAME_SIZE + 3) static DECLARE_WAIT_QUEUE_HEAD(module_wq); struct netfront_stats { <------>u64 packets; <------>u64 bytes; <------>struct u64_stats_sync syncp; }; struct netfront_info; struct netfront_queue { <------>unsigned int id; /* Queue ID, 0-based */ <------>char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */ <------>struct netfront_info *info; <------>struct bpf_prog __rcu *xdp_prog; <------>struct napi_struct napi; <------>/* Split event channels support, tx_* == rx_* when using <------> * single event channel. <------> */ <------>unsigned int tx_evtchn, rx_evtchn; <------>unsigned int tx_irq, rx_irq; <------>/* Only used when split event channels support is enabled */ <------>char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */ <------>char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */ <------>spinlock_t tx_lock; <------>struct xen_netif_tx_front_ring tx; <------>int tx_ring_ref; <------>/* <------> * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries <------> * are linked from tx_skb_freelist through tx_link. <------> */ <------>struct sk_buff *tx_skbs[NET_TX_RING_SIZE]; <------>unsigned short tx_link[NET_TX_RING_SIZE]; #define TX_LINK_NONE 0xffff #define TX_PENDING 0xfffe <------>grant_ref_t gref_tx_head; <------>grant_ref_t grant_tx_ref[NET_TX_RING_SIZE]; <------>struct page *grant_tx_page[NET_TX_RING_SIZE]; <------>unsigned tx_skb_freelist; <------>unsigned int tx_pend_queue; <------>spinlock_t rx_lock ____cacheline_aligned_in_smp; <------>struct xen_netif_rx_front_ring rx; <------>int rx_ring_ref; <------>struct timer_list rx_refill_timer; <------>struct sk_buff *rx_skbs[NET_RX_RING_SIZE]; <------>grant_ref_t gref_rx_head; <------>grant_ref_t grant_rx_ref[NET_RX_RING_SIZE]; <------>unsigned int rx_rsp_unconsumed; <------>spinlock_t rx_cons_lock; <------>struct page_pool *page_pool; <------>struct xdp_rxq_info xdp_rxq; }; struct netfront_info { <------>struct list_head list; <------>struct net_device *netdev; <------>struct xenbus_device *xbdev; <------>/* Multi-queue support */ <------>struct netfront_queue *queues; <------>/* Statistics */ <------>struct netfront_stats __percpu *rx_stats; <------>struct netfront_stats __percpu *tx_stats; <------>/* XDP state */ <------>bool netback_has_xdp_headroom; <------>bool netfront_xdp_enabled; <------>/* Is device behaving sane? */ <------>bool broken; <------>atomic_t rx_gso_checksum_fixup; }; struct netfront_rx_info { <------>struct xen_netif_rx_response rx; <------>struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1]; }; /* * Access macros for acquiring freeing slots in tx_skbs[]. */ static void add_id_to_list(unsigned *head, unsigned short *list, <------><------><------> unsigned short id) { <------>list[id] = *head; <------>*head = id; } static unsigned short get_id_from_list(unsigned *head, unsigned short *list) { <------>unsigned int id = *head; <------>if (id != TX_LINK_NONE) { <------><------>*head = list[id]; <------><------>list[id] = TX_LINK_NONE; <------>} <------>return id; } static int xennet_rxidx(RING_IDX idx) { <------>return idx & (NET_RX_RING_SIZE - 1); } static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue, <------><------><------><------><------> RING_IDX ri) { <------>int i = xennet_rxidx(ri); <------>struct sk_buff *skb = queue->rx_skbs[i]; <------>queue->rx_skbs[i] = NULL; <------>return skb; } static grant_ref_t xennet_get_rx_ref(struct netfront_queue *queue, <------><------><------><------><------> RING_IDX ri) { <------>int i = xennet_rxidx(ri); <------>grant_ref_t ref = queue->grant_rx_ref[i]; <------>queue->grant_rx_ref[i] = GRANT_INVALID_REF; <------>return ref; } #ifdef CONFIG_SYSFS static const struct attribute_group xennet_dev_group; #endif static bool xennet_can_sg(struct net_device *dev) { <------>return dev->features & NETIF_F_SG; } static void rx_refill_timeout(struct timer_list *t) { <------>struct netfront_queue *queue = from_timer(queue, t, rx_refill_timer); <------>napi_schedule(&queue->napi); } static int netfront_tx_slot_available(struct netfront_queue *queue) { <------>return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) < <------><------>(NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1); } static void xennet_maybe_wake_tx(struct netfront_queue *queue) { <------>struct net_device *dev = queue->info->netdev; <------>struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id); <------>if (unlikely(netif_tx_queue_stopped(dev_queue)) && <------> netfront_tx_slot_available(queue) && <------> likely(netif_running(dev))) <------><------>netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id)); } static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue) { <------>struct sk_buff *skb; <------>struct page *page; <------>skb = __netdev_alloc_skb(queue->info->netdev, <------><------><------><------> RX_COPY_THRESHOLD + NET_IP_ALIGN, <------><------><------><------> GFP_ATOMIC | __GFP_NOWARN); <------>if (unlikely(!skb)) <------><------>return NULL; <------>page = page_pool_dev_alloc_pages(queue->page_pool); <------>if (unlikely(!page)) { <------><------>kfree_skb(skb); <------><------>return NULL; <------>} <------>skb_add_rx_frag(skb, 0, page, 0, 0, PAGE_SIZE); <------>/* Align ip header to a 16 bytes boundary */ <------>skb_reserve(skb, NET_IP_ALIGN); <------>skb->dev = queue->info->netdev; <------>return skb; } static void xennet_alloc_rx_buffers(struct netfront_queue *queue) { <------>RING_IDX req_prod = queue->rx.req_prod_pvt; <------>int notify; <------>int err = 0; <------>if (unlikely(!netif_carrier_ok(queue->info->netdev))) <------><------>return; <------>for (req_prod = queue->rx.req_prod_pvt; <------> req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE; <------> req_prod++) { <------><------>struct sk_buff *skb; <------><------>unsigned short id; <------><------>grant_ref_t ref; <------><------>struct page *page; <------><------>struct xen_netif_rx_request *req; <------><------>skb = xennet_alloc_one_rx_buffer(queue); <------><------>if (!skb) { <------><------><------>err = -ENOMEM; <------><------><------>break; <------><------>} <------><------>id = xennet_rxidx(req_prod); <------><------>BUG_ON(queue->rx_skbs[id]); <------><------>queue->rx_skbs[id] = skb; <------><------>ref = gnttab_claim_grant_reference(&queue->gref_rx_head); <------><------>WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref)); <------><------>queue->grant_rx_ref[id] = ref; <------><------>page = skb_frag_page(&skb_shinfo(skb)->frags[0]); <------><------>req = RING_GET_REQUEST(&queue->rx, req_prod); <------><------>gnttab_page_grant_foreign_access_ref_one(ref, <------><------><------><------><------><------><------> queue->info->xbdev->otherend_id, <------><------><------><------><------><------><------> page, <------><------><------><------><------><------><------> 0); <------><------>req->id = id; <------><------>req->gref = ref; <------>} <------>queue->rx.req_prod_pvt = req_prod; <------>/* Try again later if there are not enough requests or skb allocation <------> * failed. <------> * Enough requests is quantified as the sum of newly created slots and <------> * the unconsumed slots at the backend. <------> */ <------>if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN || <------> unlikely(err)) { <------><------>mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); <------><------>return; <------>} <------>RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify); <------>if (notify) <------><------>notify_remote_via_irq(queue->rx_irq); } static int xennet_open(struct net_device *dev) { <------>struct netfront_info *np = netdev_priv(dev); <------>unsigned int num_queues = dev->real_num_tx_queues; <------>unsigned int i = 0; <------>struct netfront_queue *queue = NULL; <------>if (!np->queues || np->broken) <------><------>return -ENODEV; <------>for (i = 0; i < num_queues; ++i) { <------><------>queue = &np->queues[i]; <------><------>napi_enable(&queue->napi); <------><------>spin_lock_bh(&queue->rx_lock); <------><------>if (netif_carrier_ok(dev)) { <------><------><------>xennet_alloc_rx_buffers(queue); <------><------><------>queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1; <------><------><------>if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)) <------><------><------><------>napi_schedule(&queue->napi); <------><------>} <------><------>spin_unlock_bh(&queue->rx_lock); <------>} <------>netif_tx_start_all_queues(dev); <------>return 0; } static bool xennet_tx_buf_gc(struct netfront_queue *queue) { <------>RING_IDX cons, prod; <------>unsigned short id; <------>struct sk_buff *skb; <------>bool more_to_do; <------>bool work_done = false; <------>const struct device *dev = &queue->info->netdev->dev; <------>BUG_ON(!netif_carrier_ok(queue->info->netdev)); <------>do { <------><------>prod = queue->tx.sring->rsp_prod; <------><------>if (RING_RESPONSE_PROD_OVERFLOW(&queue->tx, prod)) { <------><------><------>dev_alert(dev, "Illegal number of responses %u\n", <------><------><------><------> prod - queue->tx.rsp_cons); <------><------><------>goto err; <------><------>} <------><------>rmb(); /* Ensure we see responses up to 'rp'. */ <------><------>for (cons = queue->tx.rsp_cons; cons != prod; cons++) { <------><------><------>struct xen_netif_tx_response txrsp; <------><------><------>work_done = true; <------><------><------>RING_COPY_RESPONSE(&queue->tx, cons, &txrsp); <------><------><------>if (txrsp.status == XEN_NETIF_RSP_NULL) <------><------><------><------>continue; <------><------><------>id = txrsp.id; <------><------><------>if (id >= RING_SIZE(&queue->tx)) { <------><------><------><------>dev_alert(dev, <------><------><------><------><------> "Response has incorrect id (%u)\n", <------><------><------><------><------> id); <------><------><------><------>goto err; <------><------><------>} <------><------><------>if (queue->tx_link[id] != TX_PENDING) { <------><------><------><------>dev_alert(dev, <------><------><------><------><------> "Response for inactive request\n"); <------><------><------><------>goto err; <------><------><------>} <------><------><------>queue->tx_link[id] = TX_LINK_NONE; <------><------><------>skb = queue->tx_skbs[id]; <------><------><------>queue->tx_skbs[id] = NULL; <------><------><------>if (unlikely(!gnttab_end_foreign_access_ref( <------><------><------><------>queue->grant_tx_ref[id], GNTMAP_readonly))) { <------><------><------><------>dev_alert(dev, <------><------><------><------><------> "Grant still in use by backend domain\n"); <------><------><------><------>goto err; <------><------><------>} <------><------><------>gnttab_release_grant_reference( <------><------><------><------>&queue->gref_tx_head, queue->grant_tx_ref[id]); <------><------><------>queue->grant_tx_ref[id] = GRANT_INVALID_REF; <------><------><------>queue->grant_tx_page[id] = NULL; <------><------><------>add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id); <------><------><------>dev_kfree_skb_irq(skb); <------><------>} <------><------>queue->tx.rsp_cons = prod; <------><------>RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do); <------>} while (more_to_do); <------>xennet_maybe_wake_tx(queue); <------>return work_done; err: <------>queue->info->broken = true; <------>dev_alert(dev, "Disabled for further use\n"); <------>return work_done; } struct xennet_gnttab_make_txreq { <------>struct netfront_queue *queue; <------>struct sk_buff *skb; <------>struct page *page; <------>struct xen_netif_tx_request *tx; /* Last request on ring page */ <------>struct xen_netif_tx_request tx_local; /* Last request local copy*/ <------>unsigned int size; }; static void xennet_tx_setup_grant(unsigned long gfn, unsigned int offset, <------><------><------><------> unsigned int len, void *data) { <------>struct xennet_gnttab_make_txreq *info = data; <------>unsigned int id; <------>struct xen_netif_tx_request *tx; <------>grant_ref_t ref; <------>/* convenient aliases */ <------>struct page *page = info->page; <------>struct netfront_queue *queue = info->queue; <------>struct sk_buff *skb = info->skb; <------>id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link); <------>tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); <------>ref = gnttab_claim_grant_reference(&queue->gref_tx_head); <------>WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref)); <------>gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, <------><------><------><------><------>gfn, GNTMAP_readonly); <------>queue->tx_skbs[id] = skb; <------>queue->grant_tx_page[id] = page; <------>queue->grant_tx_ref[id] = ref; <------>info->tx_local.id = id; <------>info->tx_local.gref = ref; <------>info->tx_local.offset = offset; <------>info->tx_local.size = len; <------>info->tx_local.flags = 0; <------>*tx = info->tx_local; <------>/* <------> * Put the request in the pending queue, it will be set to be pending <------> * when the producer index is about to be raised. <------> */ <------>add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id); <------>info->tx = tx; <------>info->size += info->tx_local.size; } static struct xen_netif_tx_request *xennet_make_first_txreq( <------>struct xennet_gnttab_make_txreq *info, <------>unsigned int offset, unsigned int len) { <------>info->size = 0; <------>gnttab_for_one_grant(info->page, offset, len, xennet_tx_setup_grant, info); <------>return info->tx; } static void xennet_make_one_txreq(unsigned long gfn, unsigned int offset, <------><------><------><------> unsigned int len, void *data) { <------>struct xennet_gnttab_make_txreq *info = data; <------>info->tx->flags |= XEN_NETTXF_more_data; <------>skb_get(info->skb); <------>xennet_tx_setup_grant(gfn, offset, len, data); } static void xennet_make_txreqs( <------>struct xennet_gnttab_make_txreq *info, <------>struct page *page, <------>unsigned int offset, unsigned int len) { <------>/* Skip unused frames from start of page */ <------>page += offset >> PAGE_SHIFT; <------>offset &= ~PAGE_MASK; <------>while (len) { <------><------>info->page = page; <------><------>info->size = 0; <------><------>gnttab_foreach_grant_in_range(page, offset, len, <------><------><------><------><------> xennet_make_one_txreq, <------><------><------><------><------> info); <------><------>page++; <------><------>offset = 0; <------><------>len -= info->size; <------>} } /* * Count how many ring slots are required to send this skb. Each frag * might be a compound page. */ static int xennet_count_skb_slots(struct sk_buff *skb) { <------>int i, frags = skb_shinfo(skb)->nr_frags; <------>int slots; <------>slots = gnttab_count_grant(offset_in_page(skb->data), <------><------><------><------> skb_headlen(skb)); <------>for (i = 0; i < frags; i++) { <------><------>skb_frag_t *frag = skb_shinfo(skb)->frags + i; <------><------>unsigned long size = skb_frag_size(frag); <------><------>unsigned long offset = skb_frag_off(frag); <------><------>/* Skip unused frames from start of page */ <------><------>offset &= ~PAGE_MASK; <------><------>slots += gnttab_count_grant(offset, size); <------>} <------>return slots; } static u16 xennet_select_queue(struct net_device *dev, struct sk_buff *skb, <------><------><------> struct net_device *sb_dev) { <------>unsigned int num_queues = dev->real_num_tx_queues; <------>u32 hash; <------>u16 queue_idx; <------>/* First, check if there is only one queue */ <------>if (num_queues == 1) { <------><------>queue_idx = 0; <------>} else { <------><------>hash = skb_get_hash(skb); <------><------>queue_idx = hash % num_queues; <------>} <------>return queue_idx; } static void xennet_mark_tx_pending(struct netfront_queue *queue) { <------>unsigned int i; <------>while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) != <------> TX_LINK_NONE) <------><------>queue->tx_link[i] = TX_PENDING; } static int xennet_xdp_xmit_one(struct net_device *dev, <------><------><------> struct netfront_queue *queue, <------><------><------> struct xdp_frame *xdpf) { <------>struct netfront_info *np = netdev_priv(dev); <------>struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); <------>struct xennet_gnttab_make_txreq info = { <------><------>.queue = queue, <------><------>.skb = NULL, <------><------>.page = virt_to_page(xdpf->data), <------>}; <------>int notify; <------>xennet_make_first_txreq(&info, <------><------><------><------>offset_in_page(xdpf->data), <------><------><------><------>xdpf->len); <------>xennet_mark_tx_pending(queue); <------>RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); <------>if (notify) <------><------>notify_remote_via_irq(queue->tx_irq); <------>u64_stats_update_begin(&tx_stats->syncp); <------>tx_stats->bytes += xdpf->len; <------>tx_stats->packets++; <------>u64_stats_update_end(&tx_stats->syncp); <------>xennet_tx_buf_gc(queue); <------>return 0; } static int xennet_xdp_xmit(struct net_device *dev, int n, <------><------><------> struct xdp_frame **frames, u32 flags) { <------>unsigned int num_queues = dev->real_num_tx_queues; <------>struct netfront_info *np = netdev_priv(dev); <------>struct netfront_queue *queue = NULL; <------>unsigned long irq_flags; <------>int drops = 0; <------>int i, err; <------>if (unlikely(np->broken)) <------><------>return -ENODEV; <------>if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) <------><------>return -EINVAL; <------>queue = &np->queues[smp_processor_id() % num_queues]; <------>spin_lock_irqsave(&queue->tx_lock, irq_flags); <------>for (i = 0; i < n; i++) { <------><------>struct xdp_frame *xdpf = frames[i]; <------><------>if (!xdpf) <------><------><------>continue; <------><------>err = xennet_xdp_xmit_one(dev, queue, xdpf); <------><------>if (err) { <------><------><------>xdp_return_frame_rx_napi(xdpf); <------><------><------>drops++; <------><------>} <------>} <------>spin_unlock_irqrestore(&queue->tx_lock, irq_flags); <------>return n - drops; } #define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1) static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev) { <------>struct netfront_info *np = netdev_priv(dev); <------>struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); <------>struct xen_netif_tx_request *first_tx; <------>unsigned int i; <------>int notify; <------>int slots; <------>struct page *page; <------>unsigned int offset; <------>unsigned int len; <------>unsigned long flags; <------>struct netfront_queue *queue = NULL; <------>struct xennet_gnttab_make_txreq info = { }; <------>unsigned int num_queues = dev->real_num_tx_queues; <------>u16 queue_index; <------>struct sk_buff *nskb; <------>/* Drop the packet if no queues are set up */ <------>if (num_queues < 1) <------><------>goto drop; <------>if (unlikely(np->broken)) <------><------>goto drop; <------>/* Determine which queue to transmit this SKB on */ <------>queue_index = skb_get_queue_mapping(skb); <------>queue = &np->queues[queue_index]; <------>/* If skb->len is too big for wire format, drop skb and alert <------> * user about misconfiguration. <------> */ <------>if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) { <------><------>net_alert_ratelimited( <------><------><------>"xennet: skb->len = %u, too big for wire format\n", <------><------><------>skb->len); <------><------>goto drop; <------>} <------>slots = xennet_count_skb_slots(skb); <------>if (unlikely(slots > MAX_XEN_SKB_FRAGS + 1)) { <------><------>net_dbg_ratelimited("xennet: skb rides the rocket: %d slots, %d bytes\n", <------><------><------><------> slots, skb->len); <------><------>if (skb_linearize(skb)) <------><------><------>goto drop; <------>} <------>page = virt_to_page(skb->data); <------>offset = offset_in_page(skb->data); <------>/* The first req should be at least ETH_HLEN size or the packet will be <------> * dropped by netback. <------> */ <------>if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) { <------><------>nskb = skb_copy(skb, GFP_ATOMIC); <------><------>if (!nskb) <------><------><------>goto drop; <------><------>dev_consume_skb_any(skb); <------><------>skb = nskb; <------><------>page = virt_to_page(skb->data); <------><------>offset = offset_in_page(skb->data); <------>} <------>len = skb_headlen(skb); <------>spin_lock_irqsave(&queue->tx_lock, flags); <------>if (unlikely(!netif_carrier_ok(dev) || <------><------> (slots > 1 && !xennet_can_sg(dev)) || <------><------> netif_needs_gso(skb, netif_skb_features(skb)))) { <------><------>spin_unlock_irqrestore(&queue->tx_lock, flags); <------><------>goto drop; <------>} <------>/* First request for the linear area. */ <------>info.queue = queue; <------>info.skb = skb; <------>info.page = page; <------>first_tx = xennet_make_first_txreq(&info, offset, len); <------>offset += info.tx_local.size; <------>if (offset == PAGE_SIZE) { <------><------>page++; <------><------>offset = 0; <------>} <------>len -= info.tx_local.size; <------>if (skb->ip_summed == CHECKSUM_PARTIAL) <------><------>/* local packet? */ <------><------>first_tx->flags |= XEN_NETTXF_csum_blank | <------><------><------><------> XEN_NETTXF_data_validated; <------>else if (skb->ip_summed == CHECKSUM_UNNECESSARY) <------><------>/* remote but checksummed. */ <------><------>first_tx->flags |= XEN_NETTXF_data_validated; <------>/* Optional extra info after the first request. */ <------>if (skb_shinfo(skb)->gso_size) { <------><------>struct xen_netif_extra_info *gso; <------><------>gso = (struct xen_netif_extra_info *) <------><------><------>RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); <------><------>first_tx->flags |= XEN_NETTXF_extra_info; <------><------>gso->u.gso.size = skb_shinfo(skb)->gso_size; <------><------>gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ? <------><------><------>XEN_NETIF_GSO_TYPE_TCPV6 : <------><------><------>XEN_NETIF_GSO_TYPE_TCPV4; <------><------>gso->u.gso.pad = 0; <------><------>gso->u.gso.features = 0; <------><------>gso->type = XEN_NETIF_EXTRA_TYPE_GSO; <------><------>gso->flags = 0; <------>} <------>/* Requests for the rest of the linear area. */ <------>xennet_make_txreqs(&info, page, offset, len); <------>/* Requests for all the frags. */ <------>for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { <------><------>skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; <------><------>xennet_make_txreqs(&info, skb_frag_page(frag), <------><------><------><------><------>skb_frag_off(frag), <------><------><------><------><------>skb_frag_size(frag)); <------>} <------>/* First request has the packet length. */ <------>first_tx->size = skb->len; <------>/* timestamp packet in software */ <------>skb_tx_timestamp(skb); <------>xennet_mark_tx_pending(queue); <------>RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); <------>if (notify) <------><------>notify_remote_via_irq(queue->tx_irq); <------>u64_stats_update_begin(&tx_stats->syncp); <------>tx_stats->bytes += skb->len; <------>tx_stats->packets++; <------>u64_stats_update_end(&tx_stats->syncp); <------>/* Note: It is not safe to access skb after xennet_tx_buf_gc()! */ <------>xennet_tx_buf_gc(queue); <------>if (!netfront_tx_slot_available(queue)) <------><------>netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); <------>spin_unlock_irqrestore(&queue->tx_lock, flags); <------>return NETDEV_TX_OK; drop: <------>dev->stats.tx_dropped++; <------>dev_kfree_skb_any(skb); <------>return NETDEV_TX_OK; } static int xennet_close(struct net_device *dev) { <------>struct netfront_info *np = netdev_priv(dev); <------>unsigned int num_queues = dev->real_num_tx_queues; <------>unsigned int i; <------>struct netfront_queue *queue; <------>netif_tx_stop_all_queues(np->netdev); <------>for (i = 0; i < num_queues; ++i) { <------><------>queue = &np->queues[i]; <------><------>napi_disable(&queue->napi); <------>} <------>return 0; } static void xennet_destroy_queues(struct netfront_info *info) { <------>unsigned int i; <------>for (i = 0; i < info->netdev->real_num_tx_queues; i++) { <------><------>struct netfront_queue *queue = &info->queues[i]; <------><------>if (netif_running(info->netdev)) <------><------><------>napi_disable(&queue->napi); <------><------>netif_napi_del(&queue->napi); <------>} <------>kfree(info->queues); <------>info->queues = NULL; } static void xennet_uninit(struct net_device *dev) { <------>struct netfront_info *np = netdev_priv(dev); <------>xennet_destroy_queues(np); } static void xennet_set_rx_rsp_cons(struct netfront_queue *queue, RING_IDX val) { <------>unsigned long flags; <------>spin_lock_irqsave(&queue->rx_cons_lock, flags); <------>queue->rx.rsp_cons = val; <------>queue->rx_rsp_unconsumed = RING_HAS_UNCONSUMED_RESPONSES(&queue->rx); <------>spin_unlock_irqrestore(&queue->rx_cons_lock, flags); } static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb, <------><------><------><------>grant_ref_t ref) { <------>int new = xennet_rxidx(queue->rx.req_prod_pvt); <------>BUG_ON(queue->rx_skbs[new]); <------>queue->rx_skbs[new] = skb; <------>queue->grant_rx_ref[new] = ref; <------>RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new; <------>RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref; <------>queue->rx.req_prod_pvt++; } static int xennet_get_extras(struct netfront_queue *queue, <------><------><------> struct xen_netif_extra_info *extras, <------><------><------> RING_IDX rp) { <------>struct xen_netif_extra_info extra; <------>struct device *dev = &queue->info->netdev->dev; <------>RING_IDX cons = queue->rx.rsp_cons; <------>int err = 0; <------>do { <------><------>struct sk_buff *skb; <------><------>grant_ref_t ref; <------><------>if (unlikely(cons + 1 == rp)) { <------><------><------>if (net_ratelimit()) <------><------><------><------>dev_warn(dev, "Missing extra info\n"); <------><------><------>err = -EBADR; <------><------><------>break; <------><------>} <------><------>RING_COPY_RESPONSE(&queue->rx, ++cons, &extra); <------><------>if (unlikely(!extra.type || <------><------><------> extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) { <------><------><------>if (net_ratelimit()) <------><------><------><------>dev_warn(dev, "Invalid extra type: %d\n", <------><------><------><------><------> extra.type); <------><------><------>err = -EINVAL; <------><------>} else { <------><------><------>extras[extra.type - 1] = extra; <------><------>} <------><------>skb = xennet_get_rx_skb(queue, cons); <------><------>ref = xennet_get_rx_ref(queue, cons); <------><------>xennet_move_rx_slot(queue, skb, ref); <------>} while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE); <------>xennet_set_rx_rsp_cons(queue, cons); <------>return err; } static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata, <------><------> struct xen_netif_rx_response *rx, struct bpf_prog *prog, <------><------> struct xdp_buff *xdp, bool *need_xdp_flush) { <------>struct xdp_frame *xdpf; <------>u32 len = rx->status; <------>u32 act; <------>int err; <------>xdp->data_hard_start = page_address(pdata); <------>xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM; <------>xdp_set_data_meta_invalid(xdp); <------>xdp->data_end = xdp->data + len; <------>xdp->rxq = &queue->xdp_rxq; <------>xdp->frame_sz = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM; <------>act = bpf_prog_run_xdp(prog, xdp); <------>switch (act) { <------>case XDP_TX: <------><------>get_page(pdata); <------><------>xdpf = xdp_convert_buff_to_frame(xdp); <------><------>err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0); <------><------>if (unlikely(err < 0)) <------><------><------>trace_xdp_exception(queue->info->netdev, prog, act); <------><------>break; <------>case XDP_REDIRECT: <------><------>get_page(pdata); <------><------>err = xdp_do_redirect(queue->info->netdev, xdp, prog); <------><------>*need_xdp_flush = true; <------><------>if (unlikely(err)) <------><------><------>trace_xdp_exception(queue->info->netdev, prog, act); <------><------>break; <------>case XDP_PASS: <------>case XDP_DROP: <------><------>break; <------>case XDP_ABORTED: <------><------>trace_xdp_exception(queue->info->netdev, prog, act); <------><------>break; <------>default: <------><------>bpf_warn_invalid_xdp_action(act); <------>} <------>return act; } static int xennet_get_responses(struct netfront_queue *queue, <------><------><------><------>struct netfront_rx_info *rinfo, RING_IDX rp, <------><------><------><------>struct sk_buff_head *list, <------><------><------><------>bool *need_xdp_flush) { <------>struct xen_netif_rx_response *rx = &rinfo->rx, rx_local; <------>int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD); <------>RING_IDX cons = queue->rx.rsp_cons; <------>struct sk_buff *skb = xennet_get_rx_skb(queue, cons); <------>struct xen_netif_extra_info *extras = rinfo->extras; <------>grant_ref_t ref = xennet_get_rx_ref(queue, cons); <------>struct device *dev = &queue->info->netdev->dev; <------>struct bpf_prog *xdp_prog; <------>struct xdp_buff xdp; <------>int slots = 1; <------>int err = 0; <------>u32 verdict; <------>if (rx->flags & XEN_NETRXF_extra_info) { <------><------>err = xennet_get_extras(queue, extras, rp); <------><------>if (!err) { <------><------><------>if (extras[XEN_NETIF_EXTRA_TYPE_XDP - 1].type) { <------><------><------><------>struct xen_netif_extra_info *xdp; <------><------><------><------>xdp = &extras[XEN_NETIF_EXTRA_TYPE_XDP - 1]; <------><------><------><------>rx->offset = xdp->u.xdp.headroom; <------><------><------>} <------><------>} <------><------>cons = queue->rx.rsp_cons; <------>} <------>for (;;) { <------><------>if (unlikely(rx->status < 0 || <------><------><------> rx->offset + rx->status > XEN_PAGE_SIZE)) { <------><------><------>if (net_ratelimit()) <------><------><------><------>dev_warn(dev, "rx->offset: %u, size: %d\n", <------><------><------><------><------> rx->offset, rx->status); <------><------><------>xennet_move_rx_slot(queue, skb, ref); <------><------><------>err = -EINVAL; <------><------><------>goto next; <------><------>} <------><------>/* <------><------> * This definitely indicates a bug, either in this driver or in <------><------> * the backend driver. In future this should flag the bad <------><------> * situation to the system controller to reboot the backend. <------><------> */ <------><------>if (ref == GRANT_INVALID_REF) { <------><------><------>if (net_ratelimit()) <------><------><------><------>dev_warn(dev, "Bad rx response id %d.\n", <------><------><------><------><------> rx->id); <------><------><------>err = -EINVAL; <------><------><------>goto next; <------><------>} <------><------>if (!gnttab_end_foreign_access_ref(ref, 0)) { <------><------><------>dev_alert(dev, <------><------><------><------> "Grant still in use by backend domain\n"); <------><------><------>queue->info->broken = true; <------><------><------>dev_alert(dev, "Disabled for further use\n"); <------><------><------>return -EINVAL; <------><------>} <------><------>gnttab_release_grant_reference(&queue->gref_rx_head, ref); <------><------>rcu_read_lock(); <------><------>xdp_prog = rcu_dereference(queue->xdp_prog); <------><------>if (xdp_prog) { <------><------><------>if (!(rx->flags & XEN_NETRXF_more_data)) { <------><------><------><------>/* currently only a single page contains data */ <------><------><------><------>verdict = xennet_run_xdp(queue, <------><------><------><------><------><------><------> skb_frag_page(&skb_shinfo(skb)->frags[0]), <------><------><------><------><------><------><------> rx, xdp_prog, &xdp, need_xdp_flush); <------><------><------><------>if (verdict != XDP_PASS) <------><------><------><------><------>err = -EINVAL; <------><------><------>} else { <------><------><------><------>/* drop the frame */ <------><------><------><------>err = -EINVAL; <------><------><------>} <------><------>} <------><------>rcu_read_unlock(); next: <------><------>__skb_queue_tail(list, skb); <------><------>if (!(rx->flags & XEN_NETRXF_more_data)) <------><------><------>break; <------><------>if (cons + slots == rp) { <------><------><------>if (net_ratelimit()) <------><------><------><------>dev_warn(dev, "Need more slots\n"); <------><------><------>err = -ENOENT; <------><------><------>break; <------><------>} <------><------>RING_COPY_RESPONSE(&queue->rx, cons + slots, &rx_local); <------><------>rx = &rx_local; <------><------>skb = xennet_get_rx_skb(queue, cons + slots); <------><------>ref = xennet_get_rx_ref(queue, cons + slots); <------><------>slots++; <------>} <------>if (unlikely(slots > max)) { <------><------>if (net_ratelimit()) <------><------><------>dev_warn(dev, "Too many slots\n"); <------><------>err = -E2BIG; <------>} <------>if (unlikely(err)) <------><------>xennet_set_rx_rsp_cons(queue, cons + slots); <------>return err; } static int xennet_set_skb_gso(struct sk_buff *skb, <------><------><------> struct xen_netif_extra_info *gso) { <------>if (!gso->u.gso.size) { <------><------>if (net_ratelimit()) <------><------><------>pr_warn("GSO size must not be zero\n"); <------><------>return -EINVAL; <------>} <------>if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 && <------> gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) { <------><------>if (net_ratelimit()) <------><------><------>pr_warn("Bad GSO type %d\n", gso->u.gso.type); <------><------>return -EINVAL; <------>} <------>skb_shinfo(skb)->gso_size = gso->u.gso.size; <------>skb_shinfo(skb)->gso_type = <------><------>(gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ? <------><------>SKB_GSO_TCPV4 : <------><------>SKB_GSO_TCPV6; <------>/* Header must be checked, and gso_segs computed. */ <------>skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; <------>skb_shinfo(skb)->gso_segs = 0; <------>return 0; } static int xennet_fill_frags(struct netfront_queue *queue, <------><------><------> struct sk_buff *skb, <------><------><------> struct sk_buff_head *list) { <------>RING_IDX cons = queue->rx.rsp_cons; <------>struct sk_buff *nskb; <------>while ((nskb = __skb_dequeue(list))) { <------><------>struct xen_netif_rx_response rx; <------><------>skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0]; <------><------>RING_COPY_RESPONSE(&queue->rx, ++cons, &rx); <------><------>if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) { <------><------><------>unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to; <------><------><------>BUG_ON(pull_to < skb_headlen(skb)); <------><------><------>__pskb_pull_tail(skb, pull_to - skb_headlen(skb)); <------><------>} <------><------>if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) { <------><------><------>xennet_set_rx_rsp_cons(queue, <------><------><------><------><------> ++cons + skb_queue_len(list)); <------><------><------>kfree_skb(nskb); <------><------><------>return -ENOENT; <------><------>} <------><------>skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, <------><------><------><------>skb_frag_page(nfrag), <------><------><------><------>rx.offset, rx.status, PAGE_SIZE); <------><------>skb_shinfo(nskb)->nr_frags = 0; <------><------>kfree_skb(nskb); <------>} <------>xennet_set_rx_rsp_cons(queue, cons); <------>return 0; } static int checksum_setup(struct net_device *dev, struct sk_buff *skb) { <------>bool recalculate_partial_csum = false; <------>/* <------> * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy <------> * peers can fail to set NETRXF_csum_blank when sending a GSO <------> * frame. In this case force the SKB to CHECKSUM_PARTIAL and <------> * recalculate the partial checksum. <------> */ <------>if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { <------><------>struct netfront_info *np = netdev_priv(dev); <------><------>atomic_inc(&np->rx_gso_checksum_fixup); <------><------>skb->ip_summed = CHECKSUM_PARTIAL; <------><------>recalculate_partial_csum = true; <------>} <------>/* A non-CHECKSUM_PARTIAL SKB does not require setup. */ <------>if (skb->ip_summed != CHECKSUM_PARTIAL) <------><------>return 0; <------>return skb_checksum_setup(skb, recalculate_partial_csum); } static int handle_incoming_queue(struct netfront_queue *queue, <------><------><------><------> struct sk_buff_head *rxq) { <------>struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats); <------>int packets_dropped = 0; <------>struct sk_buff *skb; <------>while ((skb = __skb_dequeue(rxq)) != NULL) { <------><------>int pull_to = NETFRONT_SKB_CB(skb)->pull_to; <------><------>if (pull_to > skb_headlen(skb)) <------><------><------>__pskb_pull_tail(skb, pull_to - skb_headlen(skb)); <------><------>/* Ethernet work: Delayed to here as it peeks the header. */ <------><------>skb->protocol = eth_type_trans(skb, queue->info->netdev); <------><------>skb_reset_network_header(skb); <------><------>if (checksum_setup(queue->info->netdev, skb)) { <------><------><------>kfree_skb(skb); <------><------><------>packets_dropped++; <------><------><------>queue->info->netdev->stats.rx_errors++; <------><------><------>continue; <------><------>} <------><------>u64_stats_update_begin(&rx_stats->syncp); <------><------>rx_stats->packets++; <------><------>rx_stats->bytes += skb->len; <------><------>u64_stats_update_end(&rx_stats->syncp); <------><------>/* Pass it up. */ <------><------>napi_gro_receive(&queue->napi, skb); <------>} <------>return packets_dropped; } static int xennet_poll(struct napi_struct *napi, int budget) { <------>struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi); <------>struct net_device *dev = queue->info->netdev; <------>struct sk_buff *skb; <------>struct netfront_rx_info rinfo; <------>struct xen_netif_rx_response *rx = &rinfo.rx; <------>struct xen_netif_extra_info *extras = rinfo.extras; <------>RING_IDX i, rp; <------>int work_done; <------>struct sk_buff_head rxq; <------>struct sk_buff_head errq; <------>struct sk_buff_head tmpq; <------>int err; <------>bool need_xdp_flush = false; <------>spin_lock(&queue->rx_lock); <------>skb_queue_head_init(&rxq); <------>skb_queue_head_init(&errq); <------>skb_queue_head_init(&tmpq); <------>rp = queue->rx.sring->rsp_prod; <------>if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) { <------><------>dev_alert(&dev->dev, "Illegal number of responses %u\n", <------><------><------> rp - queue->rx.rsp_cons); <------><------>queue->info->broken = true; <------><------>spin_unlock(&queue->rx_lock); <------><------>return 0; <------>} <------>rmb(); /* Ensure we see queued responses up to 'rp'. */ <------>i = queue->rx.rsp_cons; <------>work_done = 0; <------>while ((i != rp) && (work_done < budget)) { <------><------>RING_COPY_RESPONSE(&queue->rx, i, rx); <------><------>memset(extras, 0, sizeof(rinfo.extras)); <------><------>err = xennet_get_responses(queue, &rinfo, rp, &tmpq, <------><------><------><------><------> &need_xdp_flush); <------><------>if (unlikely(err)) { <------><------><------>if (queue->info->broken) { <------><------><------><------>spin_unlock(&queue->rx_lock); <------><------><------><------>return 0; <------><------><------>} err: <------><------><------>while ((skb = __skb_dequeue(&tmpq))) <------><------><------><------>__skb_queue_tail(&errq, skb); <------><------><------>dev->stats.rx_errors++; <------><------><------>i = queue->rx.rsp_cons; <------><------><------>continue; <------><------>} <------><------>skb = __skb_dequeue(&tmpq); <------><------>if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { <------><------><------>struct xen_netif_extra_info *gso; <------><------><------>gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; <------><------><------>if (unlikely(xennet_set_skb_gso(skb, gso))) { <------><------><------><------>__skb_queue_head(&tmpq, skb); <------><------><------><------>xennet_set_rx_rsp_cons(queue, <------><------><------><------><------><------> queue->rx.rsp_cons + <------><------><------><------><------><------> skb_queue_len(&tmpq)); <------><------><------><------>goto err; <------><------><------>} <------><------>} <------><------>NETFRONT_SKB_CB(skb)->pull_to = rx->status; <------><------>if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD) <------><------><------>NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD; <------><------>skb_frag_off_set(&skb_shinfo(skb)->frags[0], rx->offset); <------><------>skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status); <------><------>skb->data_len = rx->status; <------><------>skb->len += rx->status; <------><------>if (unlikely(xennet_fill_frags(queue, skb, &tmpq))) <------><------><------>goto err; <------><------>if (rx->flags & XEN_NETRXF_csum_blank) <------><------><------>skb->ip_summed = CHECKSUM_PARTIAL; <------><------>else if (rx->flags & XEN_NETRXF_data_validated) <------><------><------>skb->ip_summed = CHECKSUM_UNNECESSARY; <------><------>__skb_queue_tail(&rxq, skb); <------><------>i = queue->rx.rsp_cons + 1; <------><------>xennet_set_rx_rsp_cons(queue, i); <------><------>work_done++; <------>} <------>if (need_xdp_flush) <------><------>xdp_do_flush(); <------>__skb_queue_purge(&errq); <------>work_done -= handle_incoming_queue(queue, &rxq); <------>xennet_alloc_rx_buffers(queue); <------>if (work_done < budget) { <------><------>int more_to_do = 0; <------><------>napi_complete_done(napi, work_done); <------><------>RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do); <------><------>if (more_to_do) <------><------><------>napi_schedule(napi); <------>} <------>spin_unlock(&queue->rx_lock); <------>return work_done; } static int xennet_change_mtu(struct net_device *dev, int mtu) { <------>int max = xennet_can_sg(dev) ? XEN_NETIF_MAX_TX_SIZE : ETH_DATA_LEN; <------>if (mtu > max) <------><------>return -EINVAL; <------>dev->mtu = mtu; <------>return 0; } static void xennet_get_stats64(struct net_device *dev, <------><------><------> struct rtnl_link_stats64 *tot) { <------>struct netfront_info *np = netdev_priv(dev); <------>int cpu; <------>for_each_possible_cpu(cpu) { <------><------>struct netfront_stats *rx_stats = per_cpu_ptr(np->rx_stats, cpu); <------><------>struct netfront_stats *tx_stats = per_cpu_ptr(np->tx_stats, cpu); <------><------>u64 rx_packets, rx_bytes, tx_packets, tx_bytes; <------><------>unsigned int start; <------><------>do { <------><------><------>start = u64_stats_fetch_begin_irq(&tx_stats->syncp); <------><------><------>tx_packets = tx_stats->packets; <------><------><------>tx_bytes = tx_stats->bytes; <------><------>} while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); <------><------>do { <------><------><------>start = u64_stats_fetch_begin_irq(&rx_stats->syncp); <------><------><------>rx_packets = rx_stats->packets; <------><------><------>rx_bytes = rx_stats->bytes; <------><------>} while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); <------><------>tot->rx_packets += rx_packets; <------><------>tot->tx_packets += tx_packets; <------><------>tot->rx_bytes += rx_bytes; <------><------>tot->tx_bytes += tx_bytes; <------>} <------>tot->rx_errors = dev->stats.rx_errors; <------>tot->tx_dropped = dev->stats.tx_dropped; } static void xennet_release_tx_bufs(struct netfront_queue *queue) { <------>struct sk_buff *skb; <------>int i; <------>for (i = 0; i < NET_TX_RING_SIZE; i++) { <------><------>/* Skip over entries which are actually freelist references */ <------><------>if (!queue->tx_skbs[i]) <------><------><------>continue; <------><------>skb = queue->tx_skbs[i]; <------><------>queue->tx_skbs[i] = NULL; <------><------>get_page(queue->grant_tx_page[i]); <------><------>gnttab_end_foreign_access(queue->grant_tx_ref[i], <------><------><------><------><------> GNTMAP_readonly, <------><------><------><------><------> (unsigned long)page_address(queue->grant_tx_page[i])); <------><------>queue->grant_tx_page[i] = NULL; <------><------>queue->grant_tx_ref[i] = GRANT_INVALID_REF; <------><------>add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, i); <------><------>dev_kfree_skb_irq(skb); <------>} } static void xennet_release_rx_bufs(struct netfront_queue *queue) { <------>int id, ref; <------>spin_lock_bh(&queue->rx_lock); <------>for (id = 0; id < NET_RX_RING_SIZE; id++) { <------><------>struct sk_buff *skb; <------><------>struct page *page; <------><------>skb = queue->rx_skbs[id]; <------><------>if (!skb) <------><------><------>continue; <------><------>ref = queue->grant_rx_ref[id]; <------><------>if (ref == GRANT_INVALID_REF) <------><------><------>continue; <------><------>page = skb_frag_page(&skb_shinfo(skb)->frags[0]); <------><------>/* gnttab_end_foreign_access() needs a page ref until <------><------> * foreign access is ended (which may be deferred). <------><------> */ <------><------>get_page(page); <------><------>gnttab_end_foreign_access(ref, 0, <------><------><------><------><------> (unsigned long)page_address(page)); <------><------>queue->grant_rx_ref[id] = GRANT_INVALID_REF; <------><------>kfree_skb(skb); <------>} <------>spin_unlock_bh(&queue->rx_lock); } static netdev_features_t xennet_fix_features(struct net_device *dev, <------>netdev_features_t features) { <------>struct netfront_info *np = netdev_priv(dev); <------>if (features & NETIF_F_SG && <------> !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0)) <------><------>features &= ~NETIF_F_SG; <------>if (features & NETIF_F_IPV6_CSUM && <------> !xenbus_read_unsigned(np->xbdev->otherend, <------><------><------><------> "feature-ipv6-csum-offload", 0)) <------><------>features &= ~NETIF_F_IPV6_CSUM; <------>if (features & NETIF_F_TSO && <------> !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0)) <------><------>features &= ~NETIF_F_TSO; <------>if (features & NETIF_F_TSO6 && <------> !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0)) <------><------>features &= ~NETIF_F_TSO6; <------>return features; } static int xennet_set_features(struct net_device *dev, <------>netdev_features_t features) { <------>if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) { <------><------>netdev_info(dev, "Reducing MTU because no SG offload"); <------><------>dev->mtu = ETH_DATA_LEN; <------>} <------>return 0; } static bool xennet_handle_tx(struct netfront_queue *queue, unsigned int *eoi) { <------>unsigned long flags; <------>if (unlikely(queue->info->broken)) <------><------>return false; <------>spin_lock_irqsave(&queue->tx_lock, flags); <------>if (xennet_tx_buf_gc(queue)) <------><------>*eoi = 0; <------>spin_unlock_irqrestore(&queue->tx_lock, flags); <------>return true; } static irqreturn_t xennet_tx_interrupt(int irq, void *dev_id) { <------>unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS; <------>if (likely(xennet_handle_tx(dev_id, &eoiflag))) <------><------>xen_irq_lateeoi(irq, eoiflag); <------>return IRQ_HANDLED; } static bool xennet_handle_rx(struct netfront_queue *queue, unsigned int *eoi) { <------>unsigned int work_queued; <------>unsigned long flags; <------>if (unlikely(queue->info->broken)) <------><------>return false; <------>spin_lock_irqsave(&queue->rx_cons_lock, flags); <------>work_queued = RING_HAS_UNCONSUMED_RESPONSES(&queue->rx); <------>if (work_queued > queue->rx_rsp_unconsumed) { <------><------>queue->rx_rsp_unconsumed = work_queued; <------><------>*eoi = 0; <------>} else if (unlikely(work_queued < queue->rx_rsp_unconsumed)) { <------><------>const struct device *dev = &queue->info->netdev->dev; <------><------>spin_unlock_irqrestore(&queue->rx_cons_lock, flags); <------><------>dev_alert(dev, "RX producer index going backwards\n"); <------><------>dev_alert(dev, "Disabled for further use\n"); <------><------>queue->info->broken = true; <------><------>return false; <------>} <------>spin_unlock_irqrestore(&queue->rx_cons_lock, flags); <------>if (likely(netif_carrier_ok(queue->info->netdev) && work_queued)) <------><------>napi_schedule(&queue->napi); <------>return true; } static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id) { <------>unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS; <------>if (likely(xennet_handle_rx(dev_id, &eoiflag))) <------><------>xen_irq_lateeoi(irq, eoiflag); <------>return IRQ_HANDLED; } static irqreturn_t xennet_interrupt(int irq, void *dev_id) { <------>unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS; <------>if (xennet_handle_tx(dev_id, &eoiflag) && <------> xennet_handle_rx(dev_id, &eoiflag)) <------><------>xen_irq_lateeoi(irq, eoiflag); <------>return IRQ_HANDLED; } #ifdef CONFIG_NET_POLL_CONTROLLER static void xennet_poll_controller(struct net_device *dev) { <------>/* Poll each queue */ <------>struct netfront_info *info = netdev_priv(dev); <------>unsigned int num_queues = dev->real_num_tx_queues; <------>unsigned int i; <------>if (info->broken) <------><------>return; <------>for (i = 0; i < num_queues; ++i) <------><------>xennet_interrupt(0, &info->queues[i]); } #endif #define NETBACK_XDP_HEADROOM_DISABLE 0 #define NETBACK_XDP_HEADROOM_ENABLE 1 static int talk_to_netback_xdp(struct netfront_info *np, int xdp) { <------>int err; <------>unsigned short headroom; <------>headroom = xdp ? XDP_PACKET_HEADROOM : 0; <------>err = xenbus_printf(XBT_NIL, np->xbdev->nodename, <------><------><------> "xdp-headroom", "%hu", <------><------><------> headroom); <------>if (err) <------><------>pr_warn("Error writing xdp-headroom\n"); <------>return err; } static int xennet_xdp_set(struct net_device *dev, struct bpf_prog *prog, <------><------><------> struct netlink_ext_ack *extack) { <------>unsigned long max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM; <------>struct netfront_info *np = netdev_priv(dev); <------>struct bpf_prog *old_prog; <------>unsigned int i, err; <------>if (dev->mtu > max_mtu) { <------><------>netdev_warn(dev, "XDP requires MTU less than %lu\n", max_mtu); <------><------>return -EINVAL; <------>} <------>if (!np->netback_has_xdp_headroom) <------><------>return 0; <------>xenbus_switch_state(np->xbdev, XenbusStateReconfiguring); <------>err = talk_to_netback_xdp(np, prog ? NETBACK_XDP_HEADROOM_ENABLE : <------><------><------><------> NETBACK_XDP_HEADROOM_DISABLE); <------>if (err) <------><------>return err; <------>/* avoid the race with XDP headroom adjustment */ <------>wait_event(module_wq, <------><------> xenbus_read_driver_state(np->xbdev->otherend) == <------><------> XenbusStateReconfigured); <------>np->netfront_xdp_enabled = true; <------>old_prog = rtnl_dereference(np->queues[0].xdp_prog); <------>if (prog) <------><------>bpf_prog_add(prog, dev->real_num_tx_queues); <------>for (i = 0; i < dev->real_num_tx_queues; ++i) <------><------>rcu_assign_pointer(np->queues[i].xdp_prog, prog); <------>if (old_prog) <------><------>for (i = 0; i < dev->real_num_tx_queues; ++i) <------><------><------>bpf_prog_put(old_prog); <------>xenbus_switch_state(np->xbdev, XenbusStateConnected); <------>return 0; } static int xennet_xdp(struct net_device *dev, struct netdev_bpf *xdp) { <------>struct netfront_info *np = netdev_priv(dev); <------>if (np->broken) <------><------>return -ENODEV; <------>switch (xdp->command) { <------>case XDP_SETUP_PROG: <------><------>return xennet_xdp_set(dev, xdp->prog, xdp->extack); <------>default: <------><------>return -EINVAL; <------>} } static const struct net_device_ops xennet_netdev_ops = { <------>.ndo_uninit = xennet_uninit, <------>.ndo_open = xennet_open, <------>.ndo_stop = xennet_close, <------>.ndo_start_xmit = xennet_start_xmit, <------>.ndo_change_mtu = xennet_change_mtu, <------>.ndo_get_stats64 = xennet_get_stats64, <------>.ndo_set_mac_address = eth_mac_addr, <------>.ndo_validate_addr = eth_validate_addr, <------>.ndo_fix_features = xennet_fix_features, <------>.ndo_set_features = xennet_set_features, <------>.ndo_select_queue = xennet_select_queue, <------>.ndo_bpf = xennet_xdp, <------>.ndo_xdp_xmit = xennet_xdp_xmit, #ifdef CONFIG_NET_POLL_CONTROLLER <------>.ndo_poll_controller = xennet_poll_controller, #endif }; static void xennet_free_netdev(struct net_device *netdev) { <------>struct netfront_info *np = netdev_priv(netdev); <------>free_percpu(np->rx_stats); <------>free_percpu(np->tx_stats); <------>free_netdev(netdev); } static struct net_device *xennet_create_dev(struct xenbus_device *dev) { <------>int err; <------>struct net_device *netdev; <------>struct netfront_info *np; <------>netdev = alloc_etherdev_mq(sizeof(struct netfront_info), xennet_max_queues); <------>if (!netdev) <------><------>return ERR_PTR(-ENOMEM); <------>np = netdev_priv(netdev); <------>np->xbdev = dev; <------>np->queues = NULL; <------>err = -ENOMEM; <------>np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); <------>if (np->rx_stats == NULL) <------><------>goto exit; <------>np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); <------>if (np->tx_stats == NULL) <------><------>goto exit; <------>netdev->netdev_ops = &xennet_netdev_ops; <------>netdev->features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | <------><------><------><------> NETIF_F_GSO_ROBUST; <------>netdev->hw_features = NETIF_F_SG | <------><------><------><------> NETIF_F_IPV6_CSUM | <------><------><------><------> NETIF_F_TSO | NETIF_F_TSO6; <------>/* * Assume that all hw features are available for now. This set * will be adjusted by the call to netdev_update_features() in * xennet_connect() which is the earliest point where we can * negotiate with the backend regarding supported features. */ <------>netdev->features |= netdev->hw_features; <------>netdev->ethtool_ops = &xennet_ethtool_ops; <------>netdev->min_mtu = ETH_MIN_MTU; <------>netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE; <------>SET_NETDEV_DEV(netdev, &dev->dev); <------>np->netdev = netdev; <------>np->netfront_xdp_enabled = false; <------>netif_carrier_off(netdev); <------>do { <------><------>xenbus_switch_state(dev, XenbusStateInitialising); <------><------>err = wait_event_timeout(module_wq, <------><------><------><------> xenbus_read_driver_state(dev->otherend) != <------><------><------><------> XenbusStateClosed && <------><------><------><------> xenbus_read_driver_state(dev->otherend) != <------><------><------><------> XenbusStateUnknown, XENNET_TIMEOUT); <------>} while (!err); <------>return netdev; exit: <------>xennet_free_netdev(netdev); <------>return ERR_PTR(err); } /** * Entry point to this code when a new device is created. Allocate the basic * structures and the ring buffers for communication with the backend, and * inform the backend of the appropriate details for those. */ static int netfront_probe(struct xenbus_device *dev, <------><------><------> const struct xenbus_device_id *id) { <------>int err; <------>struct net_device *netdev; <------>struct netfront_info *info; <------>netdev = xennet_create_dev(dev); <------>if (IS_ERR(netdev)) { <------><------>err = PTR_ERR(netdev); <------><------>xenbus_dev_fatal(dev, err, "creating netdev"); <------><------>return err; <------>} <------>info = netdev_priv(netdev); <------>dev_set_drvdata(&dev->dev, info); #ifdef CONFIG_SYSFS <------>info->netdev->sysfs_groups[0] = &xennet_dev_group; #endif <------>return 0; } static void xennet_end_access(int ref, void *page) { <------>/* This frees the page as a side-effect */ <------>if (ref != GRANT_INVALID_REF) <------><------>gnttab_end_foreign_access(ref, 0, (unsigned long)page); } static void xennet_disconnect_backend(struct netfront_info *info) { <------>unsigned int i = 0; <------>unsigned int num_queues = info->netdev->real_num_tx_queues; <------>netif_carrier_off(info->netdev); <------>for (i = 0; i < num_queues && info->queues; ++i) { <------><------>struct netfront_queue *queue = &info->queues[i]; <------><------>del_timer_sync(&queue->rx_refill_timer); <------><------>if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) <------><------><------>unbind_from_irqhandler(queue->tx_irq, queue); <------><------>if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { <------><------><------>unbind_from_irqhandler(queue->tx_irq, queue); <------><------><------>unbind_from_irqhandler(queue->rx_irq, queue); <------><------>} <------><------>queue->tx_evtchn = queue->rx_evtchn = 0; <------><------>queue->tx_irq = queue->rx_irq = 0; <------><------>if (netif_running(info->netdev)) <------><------><------>napi_synchronize(&queue->napi); <------><------>xennet_release_tx_bufs(queue); <------><------>xennet_release_rx_bufs(queue); <------><------>gnttab_free_grant_references(queue->gref_tx_head); <------><------>gnttab_free_grant_references(queue->gref_rx_head); <------><------>/* End access and free the pages */ <------><------>xennet_end_access(queue->tx_ring_ref, queue->tx.sring); <------><------>xennet_end_access(queue->rx_ring_ref, queue->rx.sring); <------><------>queue->tx_ring_ref = GRANT_INVALID_REF; <------><------>queue->rx_ring_ref = GRANT_INVALID_REF; <------><------>queue->tx.sring = NULL; <------><------>queue->rx.sring = NULL; <------><------>page_pool_destroy(queue->page_pool); <------>} } /** * We are reconnecting to the backend, due to a suspend/resume, or a backend * driver restart. We tear down our netif structure and recreate it, but * leave the device-layer structures intact so that this is transparent to the * rest of the kernel. */ static int netfront_resume(struct xenbus_device *dev) { <------>struct netfront_info *info = dev_get_drvdata(&dev->dev); <------>dev_dbg(&dev->dev, "%s\n", dev->nodename); <------>netif_tx_lock_bh(info->netdev); <------>netif_device_detach(info->netdev); <------>netif_tx_unlock_bh(info->netdev); <------>xennet_disconnect_backend(info); <------>return 0; } static int xen_net_read_mac(struct xenbus_device *dev, u8 mac[]) { <------>char *s, *e, *macstr; <------>int i; <------>macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL); <------>if (IS_ERR(macstr)) <------><------>return PTR_ERR(macstr); <------>for (i = 0; i < ETH_ALEN; i++) { <------><------>mac[i] = simple_strtoul(s, &e, 16); <------><------>if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) { <------><------><------>kfree(macstr); <------><------><------>return -ENOENT; <------><------>} <------><------>s = e+1; <------>} <------>kfree(macstr); <------>return 0; } static int setup_netfront_single(struct netfront_queue *queue) { <------>int err; <------>err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); <------>if (err < 0) <------><------>goto fail; <------>err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, <------><------><------><------><------><------>xennet_interrupt, 0, <------><------><------><------><------><------>queue->info->netdev->name, <------><------><------><------><------><------>queue); <------>if (err < 0) <------><------>goto bind_fail; <------>queue->rx_evtchn = queue->tx_evtchn; <------>queue->rx_irq = queue->tx_irq = err; <------>return 0; bind_fail: <------>xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); <------>queue->tx_evtchn = 0; fail: <------>return err; } static int setup_netfront_split(struct netfront_queue *queue) { <------>int err; <------>err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); <------>if (err < 0) <------><------>goto fail; <------>err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn); <------>if (err < 0) <------><------>goto alloc_rx_evtchn_fail; <------>snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name), <------><------> "%s-tx", queue->name); <------>err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, <------><------><------><------><------><------>xennet_tx_interrupt, 0, <------><------><------><------><------><------>queue->tx_irq_name, queue); <------>if (err < 0) <------><------>goto bind_tx_fail; <------>queue->tx_irq = err; <------>snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name), <------><------> "%s-rx", queue->name); <------>err = bind_evtchn_to_irqhandler_lateeoi(queue->rx_evtchn, <------><------><------><------><------><------>xennet_rx_interrupt, 0, <------><------><------><------><------><------>queue->rx_irq_name, queue); <------>if (err < 0) <------><------>goto bind_rx_fail; <------>queue->rx_irq = err; <------>return 0; bind_rx_fail: <------>unbind_from_irqhandler(queue->tx_irq, queue); <------>queue->tx_irq = 0; bind_tx_fail: <------>xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn); <------>queue->rx_evtchn = 0; alloc_rx_evtchn_fail: <------>xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); <------>queue->tx_evtchn = 0; fail: <------>return err; } static int setup_netfront(struct xenbus_device *dev, <------><------><------>struct netfront_queue *queue, unsigned int feature_split_evtchn) { <------>struct xen_netif_tx_sring *txs; <------>struct xen_netif_rx_sring *rxs = NULL; <------>grant_ref_t gref; <------>int err; <------>queue->tx_ring_ref = GRANT_INVALID_REF; <------>queue->rx_ring_ref = GRANT_INVALID_REF; <------>queue->rx.sring = NULL; <------>queue->tx.sring = NULL; <------>txs = (struct xen_netif_tx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); <------>if (!txs) { <------><------>err = -ENOMEM; <------><------>xenbus_dev_fatal(dev, err, "allocating tx ring page"); <------><------>goto fail; <------>} <------>SHARED_RING_INIT(txs); <------>FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE); <------>err = xenbus_grant_ring(dev, txs, 1, &gref); <------>if (err < 0) <------><------>goto fail; <------>queue->tx_ring_ref = gref; <------>rxs = (struct xen_netif_rx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); <------>if (!rxs) { <------><------>err = -ENOMEM; <------><------>xenbus_dev_fatal(dev, err, "allocating rx ring page"); <------><------>goto fail; <------>} <------>SHARED_RING_INIT(rxs); <------>FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); <------>err = xenbus_grant_ring(dev, rxs, 1, &gref); <------>if (err < 0) <------><------>goto fail; <------>queue->rx_ring_ref = gref; <------>if (feature_split_evtchn) <------><------>err = setup_netfront_split(queue); <------>/* setup single event channel if <------> * a) feature-split-event-channels == 0 <------> * b) feature-split-event-channels == 1 but failed to setup <------> */ <------>if (!feature_split_evtchn || (feature_split_evtchn && err)) <------><------>err = setup_netfront_single(queue); <------>if (err) <------><------>goto fail; <------>return 0; <------>/* If we fail to setup netfront, it is safe to just revoke access to <------> * granted pages because backend is not accessing it at this point. <------> */ fail: <------>if (queue->rx_ring_ref != GRANT_INVALID_REF) { <------><------>gnttab_end_foreign_access(queue->rx_ring_ref, 0, <------><------><------><------><------> (unsigned long)rxs); <------><------>queue->rx_ring_ref = GRANT_INVALID_REF; <------>} else { <------><------>free_page((unsigned long)rxs); <------>} <------>if (queue->tx_ring_ref != GRANT_INVALID_REF) { <------><------>gnttab_end_foreign_access(queue->tx_ring_ref, 0, <------><------><------><------><------> (unsigned long)txs); <------><------>queue->tx_ring_ref = GRANT_INVALID_REF; <------>} else { <------><------>free_page((unsigned long)txs); <------>} <------>return err; } /* Queue-specific initialisation * This used to be done in xennet_create_dev() but must now * be run per-queue. */ static int xennet_init_queue(struct netfront_queue *queue) { <------>unsigned short i; <------>int err = 0; <------>char *devid; <------>spin_lock_init(&queue->tx_lock); <------>spin_lock_init(&queue->rx_lock); <------>spin_lock_init(&queue->rx_cons_lock); <------>timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0); <------>devid = strrchr(queue->info->xbdev->nodename, '/') + 1; <------>snprintf(queue->name, sizeof(queue->name), "vif%s-q%u", <------><------> devid, queue->id); <------>/* Initialise tx_skb_freelist as a free chain containing every entry. */ <------>queue->tx_skb_freelist = 0; <------>queue->tx_pend_queue = TX_LINK_NONE; <------>for (i = 0; i < NET_TX_RING_SIZE; i++) { <------><------>queue->tx_link[i] = i + 1; <------><------>queue->grant_tx_ref[i] = GRANT_INVALID_REF; <------><------>queue->grant_tx_page[i] = NULL; <------>} <------>queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE; <------>/* Clear out rx_skbs */ <------>for (i = 0; i < NET_RX_RING_SIZE; i++) { <------><------>queue->rx_skbs[i] = NULL; <------><------>queue->grant_rx_ref[i] = GRANT_INVALID_REF; <------>} <------>/* A grant for every tx ring slot */ <------>if (gnttab_alloc_grant_references(NET_TX_RING_SIZE, <------><------><------><------><------> &queue->gref_tx_head) < 0) { <------><------>pr_alert("can't alloc tx grant refs\n"); <------><------>err = -ENOMEM; <------><------>goto exit; <------>} <------>/* A grant for every rx ring slot */ <------>if (gnttab_alloc_grant_references(NET_RX_RING_SIZE, <------><------><------><------><------> &queue->gref_rx_head) < 0) { <------><------>pr_alert("can't alloc rx grant refs\n"); <------><------>err = -ENOMEM; <------><------>goto exit_free_tx; <------>} <------>return 0; exit_free_tx: <------>gnttab_free_grant_references(queue->gref_tx_head); exit: <------>return err; } static int write_queue_xenstore_keys(struct netfront_queue *queue, <------><------><------> struct xenbus_transaction *xbt, int write_hierarchical) { <------>/* Write the queue-specific keys into XenStore in the traditional <------> * way for a single queue, or in a queue subkeys for multiple <------> * queues. <------> */ <------>struct xenbus_device *dev = queue->info->xbdev; <------>int err; <------>const char *message; <------>char *path; <------>size_t pathsize; <------>/* Choose the correct place to write the keys */ <------>if (write_hierarchical) { <------><------>pathsize = strlen(dev->nodename) + 10; <------><------>path = kzalloc(pathsize, GFP_KERNEL); <------><------>if (!path) { <------><------><------>err = -ENOMEM; <------><------><------>message = "out of memory while writing ring references"; <------><------><------>goto error; <------><------>} <------><------>snprintf(path, pathsize, "%s/queue-%u", <------><------><------><------>dev->nodename, queue->id); <------>} else { <------><------>path = (char *)dev->nodename; <------>} <------>/* Write ring references */ <------>err = xenbus_printf(*xbt, path, "tx-ring-ref", "%u", <------><------><------>queue->tx_ring_ref); <------>if (err) { <------><------>message = "writing tx-ring-ref"; <------><------>goto error; <------>} <------>err = xenbus_printf(*xbt, path, "rx-ring-ref", "%u", <------><------><------>queue->rx_ring_ref); <------>if (err) { <------><------>message = "writing rx-ring-ref"; <------><------>goto error; <------>} <------>/* Write event channels; taking into account both shared <------> * and split event channel scenarios. <------> */ <------>if (queue->tx_evtchn == queue->rx_evtchn) { <------><------>/* Shared event channel */ <------><------>err = xenbus_printf(*xbt, path, <------><------><------><------>"event-channel", "%u", queue->tx_evtchn); <------><------>if (err) { <------><------><------>message = "writing event-channel"; <------><------><------>goto error; <------><------>} <------>} else { <------><------>/* Split event channels */ <------><------>err = xenbus_printf(*xbt, path, <------><------><------><------>"event-channel-tx", "%u", queue->tx_evtchn); <------><------>if (err) { <------><------><------>message = "writing event-channel-tx"; <------><------><------>goto error; <------><------>} <------><------>err = xenbus_printf(*xbt, path, <------><------><------><------>"event-channel-rx", "%u", queue->rx_evtchn); <------><------>if (err) { <------><------><------>message = "writing event-channel-rx"; <------><------><------>goto error; <------><------>} <------>} <------>if (write_hierarchical) <------><------>kfree(path); <------>return 0; error: <------>if (write_hierarchical) <------><------>kfree(path); <------>xenbus_dev_fatal(dev, err, "%s", message); <------>return err; } static int xennet_create_page_pool(struct netfront_queue *queue) { <------>int err; <------>struct page_pool_params pp_params = { <------><------>.order = 0, <------><------>.flags = 0, <------><------>.pool_size = NET_RX_RING_SIZE, <------><------>.nid = NUMA_NO_NODE, <------><------>.dev = &queue->info->netdev->dev, <------><------>.offset = XDP_PACKET_HEADROOM, <------><------>.max_len = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM, <------>}; <------>queue->page_pool = page_pool_create(&pp_params); <------>if (IS_ERR(queue->page_pool)) { <------><------>err = PTR_ERR(queue->page_pool); <------><------>queue->page_pool = NULL; <------><------>return err; <------>} <------>err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, <------><------><------> queue->id); <------>if (err) { <------><------>netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); <------><------>goto err_free_pp; <------>} <------>err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq, <------><------><------><------><------> MEM_TYPE_PAGE_POOL, queue->page_pool); <------>if (err) { <------><------>netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n"); <------><------>goto err_unregister_rxq; <------>} <------>return 0; err_unregister_rxq: <------>xdp_rxq_info_unreg(&queue->xdp_rxq); err_free_pp: <------>page_pool_destroy(queue->page_pool); <------>queue->page_pool = NULL; <------>return err; } static int xennet_create_queues(struct netfront_info *info, <------><------><------><------>unsigned int *num_queues) { <------>unsigned int i; <------>int ret; <------>info->queues = kcalloc(*num_queues, sizeof(struct netfront_queue), <------><------><------> GFP_KERNEL); <------>if (!info->queues) <------><------>return -ENOMEM; <------>for (i = 0; i < *num_queues; i++) { <------><------>struct netfront_queue *queue = &info->queues[i]; <------><------>queue->id = i; <------><------>queue->info = info; <------><------>ret = xennet_init_queue(queue); <------><------>if (ret < 0) { <------><------><------>dev_warn(&info->xbdev->dev, <------><------><------><------> "only created %d queues\n", i); <------><------><------>*num_queues = i; <------><------><------>break; <------><------>} <------><------>/* use page pool recycling instead of buddy allocator */ <------><------>ret = xennet_create_page_pool(queue); <------><------>if (ret < 0) { <------><------><------>dev_err(&info->xbdev->dev, "can't allocate page pool\n"); <------><------><------>*num_queues = i; <------><------><------>return ret; <------><------>} <------><------>netif_napi_add(queue->info->netdev, &queue->napi, <------><------><------> xennet_poll, 64); <------><------>if (netif_running(info->netdev)) <------><------><------>napi_enable(&queue->napi); <------>} <------>netif_set_real_num_tx_queues(info->netdev, *num_queues); <------>if (*num_queues == 0) { <------><------>dev_err(&info->xbdev->dev, "no queues\n"); <------><------>return -EINVAL; <------>} <------>return 0; } /* Common code used when first setting up, and when resuming. */ static int talk_to_netback(struct xenbus_device *dev, <------><------><------> struct netfront_info *info) { <------>const char *message; <------>struct xenbus_transaction xbt; <------>int err; <------>unsigned int feature_split_evtchn; <------>unsigned int i = 0; <------>unsigned int max_queues = 0; <------>struct netfront_queue *queue = NULL; <------>unsigned int num_queues = 1; <------>info->netdev->irq = 0; <------>/* Check if backend supports multiple queues */ <------>max_queues = xenbus_read_unsigned(info->xbdev->otherend, <------><------><------><------><------> "multi-queue-max-queues", 1); <------>num_queues = min(max_queues, xennet_max_queues); <------>/* Check feature-split-event-channels */ <------>feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend, <------><------><------><------><------>"feature-split-event-channels", 0); <------>/* Read mac addr. */ <------>err = xen_net_read_mac(dev, info->netdev->dev_addr); <------>if (err) { <------><------>xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename); <------><------>goto out_unlocked; <------>} <------>info->netback_has_xdp_headroom = xenbus_read_unsigned(info->xbdev->otherend, <------><------><------><------><------><------><------> "feature-xdp-headroom", 0); <------>if (info->netback_has_xdp_headroom) { <------><------>/* set the current xen-netfront xdp state */ <------><------>err = talk_to_netback_xdp(info, info->netfront_xdp_enabled ? <------><------><------><------><------> NETBACK_XDP_HEADROOM_ENABLE : <------><------><------><------><------> NETBACK_XDP_HEADROOM_DISABLE); <------><------>if (err) <------><------><------>goto out_unlocked; <------>} <------>rtnl_lock(); <------>if (info->queues) <------><------>xennet_destroy_queues(info); <------>/* For the case of a reconnect reset the "broken" indicator. */ <------>info->broken = false; <------>err = xennet_create_queues(info, &num_queues); <------>if (err < 0) { <------><------>xenbus_dev_fatal(dev, err, "creating queues"); <------><------>kfree(info->queues); <------><------>info->queues = NULL; <------><------>goto out; <------>} <------>rtnl_unlock(); <------>/* Create shared ring, alloc event channel -- for each queue */ <------>for (i = 0; i < num_queues; ++i) { <------><------>queue = &info->queues[i]; <------><------>err = setup_netfront(dev, queue, feature_split_evtchn); <------><------>if (err) <------><------><------>goto destroy_ring; <------>} again: <------>err = xenbus_transaction_start(&xbt); <------>if (err) { <------><------>xenbus_dev_fatal(dev, err, "starting transaction"); <------><------>goto destroy_ring; <------>} <------>if (xenbus_exists(XBT_NIL, <------><------><------> info->xbdev->otherend, "multi-queue-max-queues")) { <------><------>/* Write the number of queues */ <------><------>err = xenbus_printf(xbt, dev->nodename, <------><------><------><------> "multi-queue-num-queues", "%u", num_queues); <------><------>if (err) { <------><------><------>message = "writing multi-queue-num-queues"; <------><------><------>goto abort_transaction_no_dev_fatal; <------><------>} <------>} <------>if (num_queues == 1) { <------><------>err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */ <------><------>if (err) <------><------><------>goto abort_transaction_no_dev_fatal; <------>} else { <------><------>/* Write the keys for each queue */ <------><------>for (i = 0; i < num_queues; ++i) { <------><------><------>queue = &info->queues[i]; <------><------><------>err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */ <------><------><------>if (err) <------><------><------><------>goto abort_transaction_no_dev_fatal; <------><------>} <------>} <------>/* The remaining keys are not queue-specific */ <------>err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u", <------><------><------> 1); <------>if (err) { <------><------>message = "writing request-rx-copy"; <------><------>goto abort_transaction; <------>} <------>err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1); <------>if (err) { <------><------>message = "writing feature-rx-notify"; <------><------>goto abort_transaction; <------>} <------>err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1); <------>if (err) { <------><------>message = "writing feature-sg"; <------><------>goto abort_transaction; <------>} <------>err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1); <------>if (err) { <------><------>message = "writing feature-gso-tcpv4"; <------><------>goto abort_transaction; <------>} <------>err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1"); <------>if (err) { <------><------>message = "writing feature-gso-tcpv6"; <------><------>goto abort_transaction; <------>} <------>err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload", <------><------><------> "1"); <------>if (err) { <------><------>message = "writing feature-ipv6-csum-offload"; <------><------>goto abort_transaction; <------>} <------>err = xenbus_transaction_end(xbt, 0); <------>if (err) { <------><------>if (err == -EAGAIN) <------><------><------>goto again; <------><------>xenbus_dev_fatal(dev, err, "completing transaction"); <------><------>goto destroy_ring; <------>} <------>return 0; abort_transaction: <------>xenbus_dev_fatal(dev, err, "%s", message); abort_transaction_no_dev_fatal: <------>xenbus_transaction_end(xbt, 1); destroy_ring: <------>xennet_disconnect_backend(info); <------>rtnl_lock(); <------>xennet_destroy_queues(info); out: <------>rtnl_unlock(); out_unlocked: <------>device_unregister(&dev->dev); <------>return err; } static int xennet_connect(struct net_device *dev) { <------>struct netfront_info *np = netdev_priv(dev); <------>unsigned int num_queues = 0; <------>int err; <------>unsigned int j = 0; <------>struct netfront_queue *queue = NULL; <------>if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) { <------><------>dev_info(&dev->dev, <------><------><------> "backend does not support copying receive path\n"); <------><------>return -ENODEV; <------>} <------>err = talk_to_netback(np->xbdev, np); <------>if (err) <------><------>return err; <------>if (np->netback_has_xdp_headroom) <------><------>pr_info("backend supports XDP headroom\n"); <------>/* talk_to_netback() sets the correct number of queues */ <------>num_queues = dev->real_num_tx_queues; <------>if (dev->reg_state == NETREG_UNINITIALIZED) { <------><------>err = register_netdev(dev); <------><------>if (err) { <------><------><------>pr_warn("%s: register_netdev err=%d\n", __func__, err); <------><------><------>device_unregister(&np->xbdev->dev); <------><------><------>return err; <------><------>} <------>} <------>rtnl_lock(); <------>netdev_update_features(dev); <------>rtnl_unlock(); <------>/* <------> * All public and private state should now be sane. Get <------> * ready to start sending and receiving packets and give the driver <------> * domain a kick because we've probably just requeued some <------> * packets. <------> */ <------>netif_tx_lock_bh(np->netdev); <------>netif_device_attach(np->netdev); <------>netif_tx_unlock_bh(np->netdev); <------>netif_carrier_on(np->netdev); <------>for (j = 0; j < num_queues; ++j) { <------><------>queue = &np->queues[j]; <------><------>notify_remote_via_irq(queue->tx_irq); <------><------>if (queue->tx_irq != queue->rx_irq) <------><------><------>notify_remote_via_irq(queue->rx_irq); <------><------>spin_lock_irq(&queue->tx_lock); <------><------>xennet_tx_buf_gc(queue); <------><------>spin_unlock_irq(&queue->tx_lock); <------><------>spin_lock_bh(&queue->rx_lock); <------><------>xennet_alloc_rx_buffers(queue); <------><------>spin_unlock_bh(&queue->rx_lock); <------>} <------>return 0; } /** * Callback received when the backend's state changes. */ static void netback_changed(struct xenbus_device *dev, <------><------><------> enum xenbus_state backend_state) { <------>struct netfront_info *np = dev_get_drvdata(&dev->dev); <------>struct net_device *netdev = np->netdev; <------>dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state)); <------>wake_up_all(&module_wq); <------>switch (backend_state) { <------>case XenbusStateInitialising: <------>case XenbusStateInitialised: <------>case XenbusStateReconfiguring: <------>case XenbusStateReconfigured: <------>case XenbusStateUnknown: <------><------>break; <------>case XenbusStateInitWait: <------><------>if (dev->state != XenbusStateInitialising) <------><------><------>break; <------><------>if (xennet_connect(netdev) != 0) <------><------><------>break; <------><------>xenbus_switch_state(dev, XenbusStateConnected); <------><------>break; <------>case XenbusStateConnected: <------><------>netdev_notify_peers(netdev); <------><------>break; <------>case XenbusStateClosed: <------><------>if (dev->state == XenbusStateClosed) <------><------><------>break; <------><------>fallthrough; /* Missed the backend's CLOSING state */ <------>case XenbusStateClosing: <------><------>xenbus_frontend_closed(dev); <------><------>break; <------>} } static const struct xennet_stat { <------>char name[ETH_GSTRING_LEN]; <------>u16 offset; } xennet_stats[] = { <------>{ <------><------>"rx_gso_checksum_fixup", <------><------>offsetof(struct netfront_info, rx_gso_checksum_fixup) <------>}, }; static int xennet_get_sset_count(struct net_device *dev, int string_set) { <------>switch (string_set) { <------>case ETH_SS_STATS: <------><------>return ARRAY_SIZE(xennet_stats); <------>default: <------><------>return -EINVAL; <------>} } static void xennet_get_ethtool_stats(struct net_device *dev, <------><------><------><------> struct ethtool_stats *stats, u64 * data) { <------>void *np = netdev_priv(dev); <------>int i; <------>for (i = 0; i < ARRAY_SIZE(xennet_stats); i++) <------><------>data[i] = atomic_read((atomic_t *)(np + xennet_stats[i].offset)); } static void xennet_get_strings(struct net_device *dev, u32 stringset, u8 * data) { <------>int i; <------>switch (stringset) { <------>case ETH_SS_STATS: <------><------>for (i = 0; i < ARRAY_SIZE(xennet_stats); i++) <------><------><------>memcpy(data + i * ETH_GSTRING_LEN, <------><------><------> xennet_stats[i].name, ETH_GSTRING_LEN); <------><------>break; <------>} } static const struct ethtool_ops xennet_ethtool_ops = { <------>.get_link = ethtool_op_get_link, <------>.get_sset_count = xennet_get_sset_count, <------>.get_ethtool_stats = xennet_get_ethtool_stats, <------>.get_strings = xennet_get_strings, <------>.get_ts_info = ethtool_op_get_ts_info, }; #ifdef CONFIG_SYSFS static ssize_t show_rxbuf(struct device *dev, <------><------><------> struct device_attribute *attr, char *buf) { <------>return sprintf(buf, "%lu\n", NET_RX_RING_SIZE); } static ssize_t store_rxbuf(struct device *dev, <------><------><------> struct device_attribute *attr, <------><------><------> const char *buf, size_t len) { <------>char *endp; <------>unsigned long target; <------>if (!capable(CAP_NET_ADMIN)) <------><------>return -EPERM; <------>target = simple_strtoul(buf, &endp, 0); <------>if (endp == buf) <------><------>return -EBADMSG; <------>/* rxbuf_min and rxbuf_max are no longer configurable. */ <------>return len; } static DEVICE_ATTR(rxbuf_min, 0644, show_rxbuf, store_rxbuf); static DEVICE_ATTR(rxbuf_max, 0644, show_rxbuf, store_rxbuf); static DEVICE_ATTR(rxbuf_cur, 0444, show_rxbuf, NULL); static struct attribute *xennet_dev_attrs[] = { <------>&dev_attr_rxbuf_min.attr, <------>&dev_attr_rxbuf_max.attr, <------>&dev_attr_rxbuf_cur.attr, <------>NULL }; static const struct attribute_group xennet_dev_group = { <------>.attrs = xennet_dev_attrs }; #endif /* CONFIG_SYSFS */ static void xennet_bus_close(struct xenbus_device *dev) { <------>int ret; <------>if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) <------><------>return; <------>do { <------><------>xenbus_switch_state(dev, XenbusStateClosing); <------><------>ret = wait_event_timeout(module_wq, <------><------><------><------> xenbus_read_driver_state(dev->otherend) == <------><------><------><------> XenbusStateClosing || <------><------><------><------> xenbus_read_driver_state(dev->otherend) == <------><------><------><------> XenbusStateClosed || <------><------><------><------> xenbus_read_driver_state(dev->otherend) == <------><------><------><------> XenbusStateUnknown, <------><------><------><------> XENNET_TIMEOUT); <------>} while (!ret); <------>if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) <------><------>return; <------>do { <------><------>xenbus_switch_state(dev, XenbusStateClosed); <------><------>ret = wait_event_timeout(module_wq, <------><------><------><------> xenbus_read_driver_state(dev->otherend) == <------><------><------><------> XenbusStateClosed || <------><------><------><------> xenbus_read_driver_state(dev->otherend) == <------><------><------><------> XenbusStateUnknown, <------><------><------><------> XENNET_TIMEOUT); <------>} while (!ret); } static int xennet_remove(struct xenbus_device *dev) { <------>struct netfront_info *info = dev_get_drvdata(&dev->dev); <------>xennet_bus_close(dev); <------>xennet_disconnect_backend(info); <------>if (info->netdev->reg_state == NETREG_REGISTERED) <------><------>unregister_netdev(info->netdev); <------>if (info->queues) { <------><------>rtnl_lock(); <------><------>xennet_destroy_queues(info); <------><------>rtnl_unlock(); <------>} <------>xennet_free_netdev(info->netdev); <------>return 0; } static const struct xenbus_device_id netfront_ids[] = { <------>{ "vif" }, <------>{ "" } }; static struct xenbus_driver netfront_driver = { <------>.ids = netfront_ids, <------>.probe = netfront_probe, <------>.remove = xennet_remove, <------>.resume = netfront_resume, <------>.otherend_changed = netback_changed, }; static int __init netif_init(void) { <------>if (!xen_domain()) <------><------>return -ENODEV; <------>if (!xen_has_pv_nic_devices()) <------><------>return -ENODEV; <------>pr_info("Initialising Xen virtual ethernet driver\n"); <------>/* Allow as many queues as there are CPUs inut max. 8 if user has not <------> * specified a value. <------> */ <------>if (xennet_max_queues == 0) <------><------>xennet_max_queues = min_t(unsigned int, MAX_QUEUES_DEFAULT, <------><------><------><------><------> num_online_cpus()); <------>return xenbus_register_frontend(&netfront_driver); } module_init(netif_init); static void __exit netif_exit(void) { <------>xenbus_unregister_driver(&netfront_driver); } module_exit(netif_exit); MODULE_DESCRIPTION("Xen virtual network device frontend"); MODULE_LICENSE("GPL"); MODULE_ALIAS("xen:vif"); MODULE_ALIAS("xennet");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate