Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * VIRTIO based driver for vDPA device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (c) 2020, Red Hat. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *     Author: Jason Wang <jasowang@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/uuid.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <linux/virtio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <linux/vdpa.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <linux/virtio_config.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <linux/virtio_ring.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #define MOD_VERSION  "0.1"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #define MOD_AUTHOR   "Jason Wang <jasowang@redhat.com>"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #define MOD_DESC     "vDPA bus driver for virtio devices"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #define MOD_LICENSE  "GPL v2"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) struct virtio_vdpa_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	struct virtio_device vdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	struct vdpa_device *vdpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	u64 features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	/* The lock to protect virtqueue list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	/* List of virtio_vdpa_vq_info */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	struct list_head virtqueues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) struct virtio_vdpa_vq_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	/* the actual virtqueue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	struct virtqueue *vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	/* the list node for the virtqueues list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	struct list_head node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) static inline struct virtio_vdpa_device *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) to_virtio_vdpa_device(struct virtio_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	return container_of(dev, struct virtio_vdpa_device, vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	return to_virtio_vdpa_device(vdev)->vdpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) static void virtio_vdpa_get(struct virtio_device *vdev, unsigned offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 			    void *buf, unsigned len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	vdpa_get_config(vdpa, offset, buf, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) static void virtio_vdpa_set(struct virtio_device *vdev, unsigned offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 			    const void *buf, unsigned len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	ops->set_config(vdpa, offset, buf, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) static u32 virtio_vdpa_generation(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	if (ops->get_generation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 		return ops->get_generation(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	return ops->get_status(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	return ops->set_status(vdpa, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static void virtio_vdpa_reset(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	vdpa_reset(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) static bool virtio_vdpa_notify(struct virtqueue *vq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	ops->kick_vq(vdpa, vq->index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static irqreturn_t virtio_vdpa_config_cb(void *private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	struct virtio_vdpa_device *vd_dev = private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	virtio_config_changed(&vd_dev->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	struct virtio_vdpa_vq_info *info = private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	return vring_interrupt(0, info->vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static struct virtqueue *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 		     void (*callback)(struct virtqueue *vq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 		     const char *name, bool ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	struct virtio_vdpa_vq_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	struct vdpa_callback cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	struct virtqueue *vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	u64 desc_addr, driver_addr, device_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	u32 align, num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	if (!name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	if (index >= vdpa->nvqs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 		return ERR_PTR(-ENOENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	/* Queue shouldn't already be set up. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	if (ops->get_vq_ready(vdpa, index))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 		return ERR_PTR(-ENOENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	/* Allocate and fill out our active queue description */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	info = kmalloc(sizeof(*info), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 		return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	num = ops->get_vq_num_max(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	if (num == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		err = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 		goto error_new_virtqueue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	/* Create the vring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	align = ops->get_vq_align(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	vq = vring_create_virtqueue(index, num, align, vdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 				    true, true, ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 				    virtio_vdpa_notify, callback, name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	if (!vq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 		err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 		goto error_new_virtqueue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	/* Setup virtqueue callback */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	cb.callback = virtio_vdpa_virtqueue_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	cb.private = info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	ops->set_vq_cb(vdpa, index, &cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	desc_addr = virtqueue_get_desc_addr(vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	driver_addr = virtqueue_get_avail_addr(vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	device_addr = virtqueue_get_used_addr(vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	if (ops->set_vq_address(vdpa, index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 				desc_addr, driver_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 				device_addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 		err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 		goto err_vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	ops->set_vq_ready(vdpa, index, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	vq->priv = info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	info->vq = vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	spin_lock_irqsave(&vd_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	list_add(&info->node, &vd_dev->virtqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	spin_unlock_irqrestore(&vd_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	return vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) err_vq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	vring_del_virtqueue(vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) error_new_virtqueue:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	ops->set_vq_ready(vdpa, index, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	/* VDPA driver should make sure vq is stopeed here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	WARN_ON(ops->get_vq_ready(vdpa, index));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	kfree(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	return ERR_PTR(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static void virtio_vdpa_del_vq(struct virtqueue *vq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	struct vdpa_device *vdpa = vd_dev->vdpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	struct virtio_vdpa_vq_info *info = vq->priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	unsigned int index = vq->index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	spin_lock_irqsave(&vd_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	list_del(&info->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	spin_unlock_irqrestore(&vd_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	/* Select and deactivate the queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	ops->set_vq_ready(vdpa, index, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	WARN_ON(ops->get_vq_ready(vdpa, index));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	vring_del_virtqueue(vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	kfree(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	struct virtqueue *vq, *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 		virtio_vdpa_del_vq(vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned nvqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 				struct virtqueue *vqs[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 				vq_callback_t *callbacks[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 				const char * const names[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 				const bool *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 				struct irq_affinity *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	struct vdpa_callback cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	int i, err, queue_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	for (i = 0; i < nvqs; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 		if (!names[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 			vqs[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 		vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 					      callbacks[i], names[i], ctx ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 					      ctx[i] : false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		if (IS_ERR(vqs[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 			err = PTR_ERR(vqs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 			goto err_setup_vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	cb.callback = virtio_vdpa_config_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	cb.private = vd_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	ops->set_config_cb(vdpa, &cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) err_setup_vq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	virtio_vdpa_del_vqs(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	return ops->get_features(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	/* Give virtio_ring a chance to accept features. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	vring_transport_features(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	return vdpa_set_features(vdpa, vdev->features);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	struct vdpa_device *vdpa = vd_dev->vdpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	return dev_name(&vdpa->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) static const struct virtio_config_ops virtio_vdpa_config_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	.get		= virtio_vdpa_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	.set		= virtio_vdpa_set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	.generation	= virtio_vdpa_generation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	.get_status	= virtio_vdpa_get_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	.set_status	= virtio_vdpa_set_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	.reset		= virtio_vdpa_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	.find_vqs	= virtio_vdpa_find_vqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	.del_vqs	= virtio_vdpa_del_vqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	.get_features	= virtio_vdpa_get_features,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	.finalize_features = virtio_vdpa_finalize_features,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	.bus_name	= virtio_vdpa_bus_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) static void virtio_vdpa_release_dev(struct device *_d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	struct virtio_device *vdev =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	       container_of(_d, struct virtio_device, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	struct virtio_vdpa_device *vd_dev =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	       container_of(vdev, struct virtio_vdpa_device, vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	kfree(vd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) static int virtio_vdpa_probe(struct vdpa_device *vdpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 	const struct vdpa_config_ops *ops = vdpa->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 	int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	if (!vd_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 		return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 	vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	vd_dev->vdev.config = &virtio_vdpa_config_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	vd_dev->vdpa = vdpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	INIT_LIST_HEAD(&vd_dev->virtqueues);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	spin_lock_init(&vd_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	vd_dev->vdev.id.device = ops->get_device_id(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	if (vd_dev->vdev.id.device == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 		goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 	vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	ret = register_virtio_device(&vd_dev->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	reg_dev = vd_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 		goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 	vdpa_set_drvdata(vdpa, vd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	if (reg_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 		put_device(&vd_dev->vdev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 		kfree(vd_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) static void virtio_vdpa_remove(struct vdpa_device *vdpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	unregister_virtio_device(&vd_dev->vdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) static struct vdpa_driver virtio_vdpa_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	.driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 		.name	= "virtio_vdpa",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 	.probe	= virtio_vdpa_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 	.remove = virtio_vdpa_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) module_vdpa_driver(virtio_vdpa_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) MODULE_VERSION(MOD_VERSION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) MODULE_LICENSE(MOD_LICENSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) MODULE_AUTHOR(MOD_AUTHOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) MODULE_DESCRIPTION(MOD_DESC);