^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Virtio PCI driver - common functionality for all device versions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This module allows virtio devices to be used over a virtual PCI device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * This can be used with QEMU based VMMs like KVM or Xen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright IBM Corp. 2007
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright Red Hat, Inc. 2014
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Authors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Anthony Liguori <aliguori@us.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Rusty Russell <rusty@rustcorp.com.au>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Michael S. Tsirkin <mst@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "virtio_pci_common.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) static bool force_legacy = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #if IS_ENABLED(CONFIG_VIRTIO_PCI_LEGACY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) module_param(force_legacy, bool, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) MODULE_PARM_DESC(force_legacy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) "Force legacy mode for transitional virtio 1 devices");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) /* wait for pending irq handlers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) void vp_synchronize_vectors(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) if (vp_dev->intx_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) synchronize_irq(vp_dev->pci_dev->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) for (i = 0; i < vp_dev->msix_vectors; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /* the notify function used when creating a virt queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) bool vp_notify(struct virtqueue *vq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) /* we write the queue's selector into the notification register to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * signal the other end */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) iowrite16(vq->index, (void __iomem *)vq->priv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) /* Handle a configuration change: Tell driver if it wants to know. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static irqreturn_t vp_config_changed(int irq, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct virtio_pci_device *vp_dev = opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) virtio_config_changed(&vp_dev->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /* Notify all virtqueues on an interrupt. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct virtio_pci_device *vp_dev = opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct virtio_pci_vq_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) irqreturn_t ret = IRQ_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) spin_lock_irqsave(&vp_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) list_for_each_entry(info, &vp_dev->virtqueues, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) ret = IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) spin_unlock_irqrestore(&vp_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /* A small wrapper to also acknowledge the interrupt when it's handled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * I really need an EIO hook for the vring so I can ack the interrupt once we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * know that we'll be handling the IRQ but before we invoke the callback since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * the callback may notify the host which results in the host attempting to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * raise an interrupt that we would then mask once we acknowledged the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * interrupt. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static irqreturn_t vp_interrupt(int irq, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) struct virtio_pci_device *vp_dev = opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) u8 isr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /* reading the ISR has the effect of also clearing it so it's very
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * important to save off the value. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) isr = ioread8(vp_dev->isr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) /* It's definitely not us if the ISR was not high */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) if (!isr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) return IRQ_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) /* Configuration change? Tell driver if it wants to know. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) if (isr & VIRTIO_PCI_ISR_CONFIG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) vp_config_changed(irq, opaque);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) return vp_vring_interrupt(irq, opaque);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) bool per_vq_vectors, struct irq_affinity *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) const char *name = dev_name(&vp_dev->vdev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) unsigned flags = PCI_IRQ_MSIX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) unsigned i, v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) int err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) vp_dev->msix_vectors = nvectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) vp_dev->msix_names = kmalloc_array(nvectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) sizeof(*vp_dev->msix_names),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if (!vp_dev->msix_names)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) vp_dev->msix_affinity_masks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) = kcalloc(nvectors, sizeof(*vp_dev->msix_affinity_masks),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (!vp_dev->msix_affinity_masks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) for (i = 0; i < nvectors; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (desc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) flags |= PCI_IRQ_AFFINITY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) desc->pre_vectors++; /* virtio config vector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) err = pci_alloc_irq_vectors_affinity(vp_dev->pci_dev, nvectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) nvectors, flags, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) vp_dev->msix_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /* Set the vector used for configuration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) v = vp_dev->msix_used_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) "%s-config", name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) vp_config_changed, 0, vp_dev->msix_names[v],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) ++vp_dev->msix_used_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) v = vp_dev->config_vector(vp_dev, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) /* Verify we had enough resources to assign the vector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (v == VIRTIO_MSI_NO_VECTOR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (!per_vq_vectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /* Shared vector for all VQs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) v = vp_dev->msix_used_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) "%s-virtqueues", name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) vp_vring_interrupt, 0, vp_dev->msix_names[v],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) ++vp_dev->msix_used_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) void (*callback)(struct virtqueue *vq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) const char *name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) bool ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) u16 msix_vec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) struct virtqueue *vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /* fill out our structure that represents an active queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) msix_vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (IS_ERR(vq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) goto out_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) info->vq = vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (callback) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) spin_lock_irqsave(&vp_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) list_add(&info->node, &vp_dev->virtqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) spin_unlock_irqrestore(&vp_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) INIT_LIST_HEAD(&info->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) vp_dev->vqs[index] = info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) out_info:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) kfree(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) static void vp_del_vq(struct virtqueue *vq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) spin_lock_irqsave(&vp_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) list_del(&info->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) spin_unlock_irqrestore(&vp_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) vp_dev->del_vq(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) kfree(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /* the config->del_vqs() implementation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) void vp_del_vqs(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) struct virtqueue *vq, *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) if (vp_dev->per_vq_vectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) int v = vp_dev->vqs[vq->index]->msix_vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (v != VIRTIO_MSI_NO_VECTOR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) int irq = pci_irq_vector(vp_dev->pci_dev, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) irq_set_affinity_hint(irq, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) free_irq(irq, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) vp_del_vq(vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) vp_dev->per_vq_vectors = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (vp_dev->intx_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) free_irq(vp_dev->pci_dev->irq, vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) vp_dev->intx_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) for (i = 0; i < vp_dev->msix_used_vectors; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) free_irq(pci_irq_vector(vp_dev->pci_dev, i), vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (vp_dev->msix_affinity_masks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) for (i = 0; i < vp_dev->msix_vectors; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (vp_dev->msix_affinity_masks[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) free_cpumask_var(vp_dev->msix_affinity_masks[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (vp_dev->msix_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /* Disable the vector used for configuration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) pci_free_irq_vectors(vp_dev->pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) vp_dev->msix_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) vp_dev->msix_vectors = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) vp_dev->msix_used_vectors = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) kfree(vp_dev->msix_names);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) vp_dev->msix_names = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) kfree(vp_dev->msix_affinity_masks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) vp_dev->msix_affinity_masks = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) kfree(vp_dev->vqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) vp_dev->vqs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct virtqueue *vqs[], vq_callback_t *callbacks[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) const char * const names[], bool per_vq_vectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) const bool *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct irq_affinity *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) u16 msix_vec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) int i, err, nvectors, allocated_vectors, queue_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (!vp_dev->vqs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) if (per_vq_vectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) /* Best option: one for change interrupt, one per vq. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) nvectors = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) for (i = 0; i < nvqs; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (names[i] && callbacks[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) ++nvectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /* Second best: one for change, shared for all vqs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) nvectors = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) per_vq_vectors ? desc : NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) goto error_find;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) vp_dev->per_vq_vectors = per_vq_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) allocated_vectors = vp_dev->msix_used_vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) for (i = 0; i < nvqs; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (!names[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) vqs[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (!callbacks[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) msix_vec = VIRTIO_MSI_NO_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) else if (vp_dev->per_vq_vectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) msix_vec = allocated_vectors++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) msix_vec = VP_MSIX_VQ_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) ctx ? ctx[i] : false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) msix_vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (IS_ERR(vqs[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) err = PTR_ERR(vqs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) goto error_find;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) /* allocate per-vq irq if available and necessary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) snprintf(vp_dev->msix_names[msix_vec],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) sizeof *vp_dev->msix_names,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) "%s-%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) dev_name(&vp_dev->vdev.dev), names[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) vring_interrupt, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) vp_dev->msix_names[msix_vec],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) vqs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) goto error_find;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) error_find:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) vp_del_vqs(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned nvqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) struct virtqueue *vqs[], vq_callback_t *callbacks[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) const char * const names[], const bool *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) int i, err, queue_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (!vp_dev->vqs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) dev_name(&vdev->dev), vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) goto out_del_vqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) vp_dev->intx_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) vp_dev->per_vq_vectors = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) for (i = 0; i < nvqs; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (!names[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) vqs[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) ctx ? ctx[i] : false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) VIRTIO_MSI_NO_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if (IS_ERR(vqs[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) err = PTR_ERR(vqs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) goto out_del_vqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) out_del_vqs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) vp_del_vqs(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) /* the config->find_vqs() implementation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) struct virtqueue *vqs[], vq_callback_t *callbacks[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) const char * const names[], const bool *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) struct irq_affinity *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /* Try MSI-X with one vector per queue. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true, ctx, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /* Fallback: MSI-X with one vector for config, one shared for queues. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false, ctx, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) /* Finally fall back to regular interrupts. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) const char *vp_bus_name(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) return pci_name(vp_dev->pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) /* Setup the affinity for a virtqueue:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * - force the affinity for per vq vector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * - OR over all affinities for shared MSI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * - ignore the affinity request if we're using INTX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) int vp_set_vq_affinity(struct virtqueue *vq, const struct cpumask *cpu_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) struct virtio_device *vdev = vq->vdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct cpumask *mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) unsigned int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (!vq->callback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (vp_dev->msix_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) mask = vp_dev->msix_affinity_masks[info->msix_vector];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) irq = pci_irq_vector(vp_dev->pci_dev, info->msix_vector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) if (!cpu_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) irq_set_affinity_hint(irq, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) cpumask_copy(mask, cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) irq_set_affinity_hint(irq, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) const struct cpumask *vp_get_vq_affinity(struct virtio_device *vdev, int index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) if (!vp_dev->per_vq_vectors ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) vp_dev->vqs[index]->msix_vector == VIRTIO_MSI_NO_VECTOR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) return pci_irq_get_affinity(vp_dev->pci_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) vp_dev->vqs[index]->msix_vector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) #ifdef CONFIG_PM_SLEEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) static int virtio_pci_freeze(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) struct pci_dev *pci_dev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) ret = virtio_device_freeze(&vp_dev->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) pci_disable_device(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) static int virtio_pci_restore(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) struct pci_dev *pci_dev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) ret = pci_enable_device(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) pci_set_master(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return virtio_device_restore(&vp_dev->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) static const struct dev_pm_ops virtio_pci_pm_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) static const struct pci_device_id virtio_pci_id_table[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) { 0 }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) static void virtio_pci_release_dev(struct device *_d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) struct virtio_device *vdev = dev_to_virtio(_d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) struct virtio_pci_device *vp_dev = to_vp_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) /* As struct device is a kobject, it's not safe to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * free the memory (including the reference counter itself)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * until it's release callback. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) kfree(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) static int virtio_pci_probe(struct pci_dev *pci_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) const struct pci_device_id *id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct virtio_pci_device *vp_dev, *reg_dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) /* allocate our structure and fill it out */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) if (!vp_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) pci_set_drvdata(pci_dev, vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) vp_dev->vdev.dev.parent = &pci_dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) vp_dev->vdev.dev.release = virtio_pci_release_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) vp_dev->pci_dev = pci_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) INIT_LIST_HEAD(&vp_dev->virtqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) spin_lock_init(&vp_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /* enable the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) rc = pci_enable_device(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) goto err_enable_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (force_legacy) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) rc = virtio_pci_legacy_probe(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) /* Also try modern mode if we can't map BAR0 (no IO space). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (rc == -ENODEV || rc == -ENOMEM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) rc = virtio_pci_modern_probe(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) goto err_probe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) rc = virtio_pci_modern_probe(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) if (rc == -ENODEV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) rc = virtio_pci_legacy_probe(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) goto err_probe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) pci_set_master(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) rc = register_virtio_device(&vp_dev->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) reg_dev = vp_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) goto err_register;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) err_register:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (vp_dev->ioaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) virtio_pci_legacy_remove(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) virtio_pci_modern_remove(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) err_probe:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) pci_disable_device(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) err_enable_device:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (reg_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) put_device(&vp_dev->vdev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) kfree(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) static void virtio_pci_remove(struct pci_dev *pci_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) struct device *dev = get_device(&vp_dev->vdev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) * Device is marked broken on surprise removal so that virtio upper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * layers can abort any ongoing operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (!pci_device_is_present(pci_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) virtio_break_device(&vp_dev->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) pci_disable_sriov(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) unregister_virtio_device(&vp_dev->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (vp_dev->ioaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) virtio_pci_legacy_remove(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) virtio_pci_modern_remove(vp_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) pci_disable_device(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) put_device(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) static int virtio_pci_sriov_configure(struct pci_dev *pci_dev, int num_vfs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) struct virtio_device *vdev = &vp_dev->vdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) if (!(vdev->config->get_status(vdev) & VIRTIO_CONFIG_S_DRIVER_OK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (!__virtio_test_bit(vdev, VIRTIO_F_SR_IOV))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) if (pci_vfs_assigned(pci_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if (num_vfs == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) pci_disable_sriov(pci_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) ret = pci_enable_sriov(pci_dev, num_vfs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) return num_vfs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) static struct pci_driver virtio_pci_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) .name = "virtio-pci",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) .id_table = virtio_pci_id_table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) .probe = virtio_pci_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) .remove = virtio_pci_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) #ifdef CONFIG_PM_SLEEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) .driver.pm = &virtio_pci_pm_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) .sriov_configure = virtio_pci_sriov_configure,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) module_pci_driver(virtio_pci_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) MODULE_DESCRIPTION("virtio-pci");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) MODULE_VERSION("1");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags