Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #ifndef _VHOST_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #define _VHOST_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/eventfd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/vhost.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/poll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/uio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/virtio_config.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/virtio_ring.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/vhost_iotlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <linux/irqbypass.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) struct vhost_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) typedef void (*vhost_work_fn_t)(struct vhost_work *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #define VHOST_WORK_QUEUED 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) struct vhost_work {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	struct llist_node	  node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	vhost_work_fn_t		  fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	unsigned long		  flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) /* Poll a file (eventfd or socket) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) /* Note: there's nothing vhost specific about this structure. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) struct vhost_poll {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	poll_table                table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	wait_queue_head_t        *wqh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	wait_queue_entry_t              wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	struct vhost_work	  work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	__poll_t		  mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	struct vhost_dev	 *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) bool vhost_has_work(struct vhost_dev *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 		     __poll_t mask, struct vhost_dev *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) int vhost_poll_start(struct vhost_poll *poll, struct file *file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) void vhost_poll_stop(struct vhost_poll *poll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) void vhost_poll_flush(struct vhost_poll *poll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) void vhost_poll_queue(struct vhost_poll *poll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) struct vhost_log {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	u64 addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	u64 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) enum vhost_uaddr_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	VHOST_ADDR_DESC = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	VHOST_ADDR_AVAIL = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	VHOST_ADDR_USED = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	VHOST_NUM_ADDRS = 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) struct vhost_vring_call {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	struct eventfd_ctx *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	struct irq_bypass_producer producer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) /* The virtqueue structure describes a queue attached to a device. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) struct vhost_virtqueue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	struct vhost_dev *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	/* The actual ring of buffers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	struct mutex mutex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	unsigned int num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	vring_desc_t __user *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	vring_avail_t __user *avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	vring_used_t __user *used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	const struct vhost_iotlb_map *meta_iotlb[VHOST_NUM_ADDRS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	struct file *kick;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	struct vhost_vring_call call_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	struct eventfd_ctx *error_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	struct eventfd_ctx *log_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	struct vhost_poll poll;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	/* The routine to call when the Guest pings us, or timeout. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	vhost_work_fn_t handle_kick;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	/* Last available index we saw. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	u16 last_avail_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	/* Caches available index value from user. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	u16 avail_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	/* Last index we used. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	u16 last_used_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	/* Used flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	u16 used_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	/* Last used index value we have signalled on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	u16 signalled_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	/* Last used index value we have signalled on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	bool signalled_used_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	/* Log writes to used structure. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	bool log_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	u64 log_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	struct iovec iov[UIO_MAXIOV];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	struct iovec iotlb_iov[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	struct iovec *indirect;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	struct vring_used_elem *heads;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	/* Protected by virtqueue mutex. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	struct vhost_iotlb *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	struct vhost_iotlb *iotlb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	void *private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	u64 acked_features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	u64 acked_backend_features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	/* Log write descriptors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	void __user *log_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	struct vhost_log *log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	struct iovec log_iov[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	/* Ring endianness. Defaults to legacy native endianness.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	 * Set to true when starting a modern virtio device. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	bool is_le;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	/* Ring endianness requested by userspace for cross-endian support. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	bool user_be;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	u32 busyloop_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct vhost_msg_node {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)   union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	  struct vhost_msg msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	  struct vhost_msg_v2 msg_v2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)   };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)   struct vhost_virtqueue *vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)   struct list_head node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct vhost_dev {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	struct mutex mutex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	struct vhost_virtqueue **vqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	int nvqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	struct eventfd_ctx *log_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	struct llist_head work_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	struct task_struct *worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	struct vhost_iotlb *umem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	struct vhost_iotlb *iotlb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	spinlock_t iotlb_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	struct list_head read_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	struct list_head pending_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	wait_queue_head_t wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	int iov_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	int weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	int byte_weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	u64 kcov_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	bool use_worker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	int (*msg_handler)(struct vhost_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 			   struct vhost_iotlb_msg *msg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) bool vhost_exceeds_weight(struct vhost_virtqueue *vq, int pkts, int total_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 		    int nvqs, int iov_limit, int weight, int byte_weight,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 		    bool use_worker,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 		    int (*msg_handler)(struct vhost_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 				       struct vhost_iotlb_msg *msg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) long vhost_dev_set_owner(struct vhost_dev *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) bool vhost_dev_has_owner(struct vhost_dev *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) long vhost_dev_check_owner(struct vhost_dev *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) struct vhost_iotlb *vhost_dev_reset_owner_prepare(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) void vhost_dev_cleanup(struct vhost_dev *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) void vhost_dev_stop(struct vhost_dev *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) bool vhost_vq_access_ok(struct vhost_virtqueue *vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) bool vhost_log_access_ok(struct vhost_dev *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) int vhost_get_vq_desc(struct vhost_virtqueue *,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		      struct iovec iov[], unsigned int iov_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 		      unsigned int *out_num, unsigned int *in_num,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 		      struct vhost_log *log, unsigned int *log_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) void vhost_discard_vq_desc(struct vhost_virtqueue *, int n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) bool vhost_vq_is_setup(struct vhost_virtqueue *vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) int vhost_vq_init_access(struct vhost_virtqueue *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) int vhost_add_used(struct vhost_virtqueue *, unsigned int head, int len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) int vhost_add_used_n(struct vhost_virtqueue *, struct vring_used_elem *heads,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 		     unsigned count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) void vhost_add_used_and_signal(struct vhost_dev *, struct vhost_virtqueue *,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 			       unsigned int id, int len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) void vhost_add_used_and_signal_n(struct vhost_dev *, struct vhost_virtqueue *,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 			       struct vring_used_elem *heads, unsigned count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) bool vhost_vq_avail_empty(struct vhost_dev *, struct vhost_virtqueue *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 		    unsigned int log_num, u64 len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 		    struct iovec *iov, int count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) int vq_meta_prefetch(struct vhost_virtqueue *vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) void vhost_enqueue_msg(struct vhost_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 		       struct list_head *head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		       struct vhost_msg_node *node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 					 struct list_head *head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) void vhost_set_backend_features(struct vhost_dev *dev, u64 features);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) __poll_t vhost_chr_poll(struct file *file, struct vhost_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 			    poll_table *wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 			    int noblock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 			     struct iov_iter *from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 			  struct vhost_iotlb_map *map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) #define vq_err(vq, fmt, ...) do {                                  \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 		pr_debug(pr_fmt(fmt), ##__VA_ARGS__);       \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 		if ((vq)->error_ctx)                               \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 				eventfd_signal((vq)->error_ctx, 1);\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	} while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	VHOST_FEATURES = (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 			 (1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 			 (1ULL << VIRTIO_RING_F_EVENT_IDX) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 			 (1ULL << VHOST_F_LOG_ALL) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 			 (1ULL << VIRTIO_F_ANY_LAYOUT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 			 (1ULL << VIRTIO_F_VERSION_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)  * vhost_vq_set_backend - Set backend.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)  * @vq            Virtqueue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)  * @private_data  The private data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)  * Context: Need to call with vq->mutex acquired.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) static inline void vhost_vq_set_backend(struct vhost_virtqueue *vq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 					void *private_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	vq->private_data = private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)  * vhost_vq_get_backend - Get backend.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)  * @vq            Virtqueue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)  * Context: Need to call with vq->mutex acquired.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)  * Return: Private data previously set with vhost_vq_set_backend.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) static inline void *vhost_vq_get_backend(struct vhost_virtqueue *vq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	return vq->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) static inline bool vhost_has_feature(struct vhost_virtqueue *vq, int bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	return vq->acked_features & (1ULL << bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) static inline bool vhost_backend_has_feature(struct vhost_virtqueue *vq, int bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	return vq->acked_backend_features & (1ULL << bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static inline bool vhost_is_little_endian(struct vhost_virtqueue *vq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	return vq->is_le;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static inline bool vhost_is_little_endian(struct vhost_virtqueue *vq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	return virtio_legacy_is_little_endian() || vq->is_le;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /* Memory accessors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) static inline u16 vhost16_to_cpu(struct vhost_virtqueue *vq, __virtio16 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	return __virtio16_to_cpu(vhost_is_little_endian(vq), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) static inline __virtio16 cpu_to_vhost16(struct vhost_virtqueue *vq, u16 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	return __cpu_to_virtio16(vhost_is_little_endian(vq), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) static inline u32 vhost32_to_cpu(struct vhost_virtqueue *vq, __virtio32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	return __virtio32_to_cpu(vhost_is_little_endian(vq), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) static inline __virtio32 cpu_to_vhost32(struct vhost_virtqueue *vq, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	return __cpu_to_virtio32(vhost_is_little_endian(vq), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) static inline u64 vhost64_to_cpu(struct vhost_virtqueue *vq, __virtio64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	return __virtio64_to_cpu(vhost_is_little_endian(vq), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static inline __virtio64 cpu_to_vhost64(struct vhost_virtqueue *vq, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	return __cpu_to_virtio64(vhost_is_little_endian(vq), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) #endif