^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * vhost transport for vsock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2013-2015 Red Hat, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Author: Asias He <asias@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Stefan Hajnoczi <stefanha@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/miscdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <net/sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/virtio_vsock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/vhost.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/hashtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <net/af_vsock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "vhost.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define VHOST_VSOCK_DEFAULT_HOST_CID 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) /* Max number of bytes transferred before requeueing the job.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * Using this limit prevents one virtqueue from starving others. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define VHOST_VSOCK_WEIGHT 0x80000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /* Max number of packets transferred before requeueing the job.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * Using this limit prevents one virtqueue from starving others with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * small pkts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define VHOST_VSOCK_PKT_WEIGHT 256
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) VHOST_VSOCK_FEATURES = VHOST_FEATURES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /* Used to track all the vhost_vsock instances on the system. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) static DEFINE_MUTEX(vhost_vsock_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct vhost_vsock {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) struct vhost_dev dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct vhost_virtqueue vqs[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct hlist_node hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct vhost_work send_pkt_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) spinlock_t send_pkt_list_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct list_head send_pkt_list; /* host->guest pending packets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) atomic_t queued_replies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) u32 guest_cid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) static u32 vhost_transport_get_local_cid(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) return VHOST_VSOCK_DEFAULT_HOST_CID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) /* Callers that dereference the return value must hold vhost_vsock_mutex or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * RCU read lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static struct vhost_vsock *vhost_vsock_get(u32 guest_cid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct vhost_vsock *vsock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) u32 other_cid = vsock->guest_cid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /* Skip instances that have no CID yet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (other_cid == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (other_cid == guest_cid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return vsock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) vhost_transport_do_send_pkt(struct vhost_vsock *vsock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct vhost_virtqueue *vq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) int pkts = 0, total_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) bool added = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) bool restart_tx = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) mutex_lock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) if (!vhost_vq_get_backend(vq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) /* Avoid further vmexits, we're already processing the virtqueue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) vhost_disable_notify(&vsock->dev, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct virtio_vsock_pkt *pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct iov_iter iov_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) unsigned out, in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) size_t nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) size_t iov_len, payload_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) int head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) spin_lock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (list_empty(&vsock->send_pkt_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) spin_unlock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) vhost_enable_notify(&vsock->dev, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) pkt = list_first_entry(&vsock->send_pkt_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct virtio_vsock_pkt, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) list_del_init(&pkt->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) spin_unlock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) &out, &in, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (head < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) spin_lock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) list_add(&pkt->list, &vsock->send_pkt_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) spin_unlock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (head == vq->num) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) spin_lock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) list_add(&pkt->list, &vsock->send_pkt_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) spin_unlock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /* We cannot finish yet if more buffers snuck in while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * re-enabling notify.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) vhost_disable_notify(&vsock->dev, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) if (out) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) vq_err(vq, "Expected 0 output buffers, got %u\n", out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) iov_len = iov_length(&vq->iov[out], in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (iov_len < sizeof(pkt->hdr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) vq_err(vq, "Buffer len [%zu] too small\n", iov_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) iov_iter_init(&iov_iter, READ, &vq->iov[out], in, iov_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) payload_len = pkt->len - pkt->off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) /* If the packet is greater than the space available in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * buffer, we split it using multiple buffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (payload_len > iov_len - sizeof(pkt->hdr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) payload_len = iov_len - sizeof(pkt->hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) /* Set the correct length in the header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) pkt->hdr.len = cpu_to_le32(payload_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) nbytes = copy_to_iter(&pkt->hdr, sizeof(pkt->hdr), &iov_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (nbytes != sizeof(pkt->hdr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) vq_err(vq, "Faulted on copying pkt hdr\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) nbytes = copy_to_iter(pkt->buf + pkt->off, payload_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) &iov_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (nbytes != payload_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) vq_err(vq, "Faulted on copying pkt buf\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) /* Deliver to monitoring devices all packets that we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * will transmit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) virtio_transport_deliver_tap_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) vhost_add_used(vq, head, sizeof(pkt->hdr) + payload_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) added = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) pkt->off += payload_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) total_len += payload_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /* If we didn't send all the payload we can requeue the packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * to send it with the next available buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (pkt->off < pkt->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* We are queueing the same virtio_vsock_pkt to handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * the remaining bytes, and we want to deliver it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * to monitoring devices in the next iteration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) pkt->tap_delivered = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) spin_lock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) list_add(&pkt->list, &vsock->send_pkt_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) spin_unlock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if (pkt->reply) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) int val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) val = atomic_dec_return(&vsock->queued_replies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /* Do we have resources to resume tx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * processing?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (val + 1 == tx_vq->num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) restart_tx = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) if (added)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) vhost_signal(&vsock->dev, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) mutex_unlock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (restart_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) vhost_poll_queue(&tx_vq->poll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) static void vhost_transport_send_pkt_work(struct vhost_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) struct vhost_virtqueue *vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) struct vhost_vsock *vsock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) vsock = container_of(work, struct vhost_vsock, send_pkt_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) vq = &vsock->vqs[VSOCK_VQ_RX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) vhost_transport_do_send_pkt(vsock, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) vhost_transport_send_pkt(struct virtio_vsock_pkt *pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) struct vhost_vsock *vsock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) int len = pkt->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /* Find the vhost_vsock according to guest context id */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) vsock = vhost_vsock_get(le64_to_cpu(pkt->hdr.dst_cid));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (!vsock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (pkt->reply)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) atomic_inc(&vsock->queued_replies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) spin_lock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) list_add_tail(&pkt->list, &vsock->send_pkt_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) spin_unlock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) vhost_work_queue(&vsock->dev, &vsock->send_pkt_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) vhost_transport_cancel_pkt(struct vsock_sock *vsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) struct vhost_vsock *vsock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct virtio_vsock_pkt *pkt, *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) int cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) int ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) LIST_HEAD(freeme);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /* Find the vhost_vsock according to guest context id */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) vsock = vhost_vsock_get(vsk->remote_addr.svm_cid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) if (!vsock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) spin_lock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (pkt->vsk != vsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) list_move(&pkt->list, &freeme);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) spin_unlock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) list_for_each_entry_safe(pkt, n, &freeme, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) if (pkt->reply)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) list_del(&pkt->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) int new_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (new_cnt + cnt >= tx_vq->num && new_cnt < tx_vq->num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) vhost_poll_queue(&tx_vq->poll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) static struct virtio_vsock_pkt *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) unsigned int out, unsigned int in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) struct virtio_vsock_pkt *pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) struct iov_iter iov_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) size_t nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) size_t len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (in != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) vq_err(vq, "Expected 0 input buffers, got %u\n", in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) pkt = kzalloc(sizeof(*pkt), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (!pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) len = iov_length(vq->iov, out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) iov_iter_init(&iov_iter, WRITE, vq->iov, out, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) nbytes = copy_from_iter(&pkt->hdr, sizeof(pkt->hdr), &iov_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (nbytes != sizeof(pkt->hdr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) sizeof(pkt->hdr), nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) kfree(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) if (le16_to_cpu(pkt->hdr.type) == VIRTIO_VSOCK_TYPE_STREAM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) pkt->len = le32_to_cpu(pkt->hdr.len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) /* No payload */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) if (!pkt->len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) return pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) /* The pkt is too big */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (pkt->len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) kfree(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) pkt->buf = kmalloc(pkt->len, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) if (!pkt->buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) kfree(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) pkt->buf_len = pkt->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) nbytes = copy_from_iter(pkt->buf, pkt->len, &iov_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (nbytes != pkt->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) vq_err(vq, "Expected %u byte payload, got %zu bytes\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) pkt->len, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) return pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) /* Is there space left for replies to rx packets? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) static bool vhost_vsock_more_replies(struct vhost_vsock *vsock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) int val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) val = atomic_read(&vsock->queued_replies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) return val < vq->num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) static struct virtio_transport vhost_transport = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) .transport = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) .module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) .get_local_cid = vhost_transport_get_local_cid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) .init = virtio_transport_do_socket_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) .destruct = virtio_transport_destruct,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) .release = virtio_transport_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) .connect = virtio_transport_connect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) .shutdown = virtio_transport_shutdown,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) .cancel_pkt = vhost_transport_cancel_pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) .dgram_enqueue = virtio_transport_dgram_enqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) .dgram_dequeue = virtio_transport_dgram_dequeue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) .dgram_bind = virtio_transport_dgram_bind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) .dgram_allow = virtio_transport_dgram_allow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) .stream_enqueue = virtio_transport_stream_enqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) .stream_dequeue = virtio_transport_stream_dequeue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) .stream_has_data = virtio_transport_stream_has_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) .stream_has_space = virtio_transport_stream_has_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) .stream_rcvhiwat = virtio_transport_stream_rcvhiwat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) .stream_is_active = virtio_transport_stream_is_active,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) .stream_allow = virtio_transport_stream_allow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) .notify_poll_in = virtio_transport_notify_poll_in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) .notify_poll_out = virtio_transport_notify_poll_out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) .notify_recv_init = virtio_transport_notify_recv_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) .notify_recv_pre_block = virtio_transport_notify_recv_pre_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) .notify_send_init = virtio_transport_notify_send_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) .notify_send_pre_block = virtio_transport_notify_send_pre_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) .notify_buffer_size = virtio_transport_notify_buffer_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) .send_pkt = vhost_transport_send_pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) static void vhost_vsock_handle_tx_kick(struct vhost_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) poll.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct virtio_vsock_pkt *pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) int head, pkts = 0, total_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) unsigned int out, in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) bool added = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) mutex_lock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (!vhost_vq_get_backend(vq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) vhost_disable_notify(&vsock->dev, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) u32 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (!vhost_vsock_more_replies(vsock)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) /* Stop tx until the device processes already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * pending replies. Leave tx virtqueue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * callbacks disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) goto no_more_replies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) &out, &in, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) if (head < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if (head == vq->num) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) vhost_disable_notify(&vsock->dev, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) pkt = vhost_vsock_alloc_pkt(vq, out, in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (!pkt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) vq_err(vq, "Faulted on pkt\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) len = pkt->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) /* Deliver to monitoring devices all received packets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) virtio_transport_deliver_tap_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /* Only accept correctly addressed packets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (le64_to_cpu(pkt->hdr.src_cid) == vsock->guest_cid &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) le64_to_cpu(pkt->hdr.dst_cid) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) vhost_transport_get_local_cid())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) virtio_transport_recv_pkt(&vhost_transport, pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) len += sizeof(pkt->hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) vhost_add_used(vq, head, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) total_len += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) added = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) no_more_replies:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (added)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) vhost_signal(&vsock->dev, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) mutex_unlock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) static void vhost_vsock_handle_rx_kick(struct vhost_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) poll.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) vhost_transport_do_send_pkt(vsock, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) static int vhost_vsock_start(struct vhost_vsock *vsock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) struct vhost_virtqueue *vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) size_t i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) mutex_lock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) ret = vhost_dev_check_owner(&vsock->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) vq = &vsock->vqs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) mutex_lock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (!vhost_vq_access_ok(vq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) goto err_vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (!vhost_vq_get_backend(vq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) vhost_vq_set_backend(vq, vsock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) ret = vhost_vq_init_access(vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) goto err_vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) mutex_unlock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) /* Some packets may have been queued before the device was started,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * let's kick the send worker to send them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) vhost_work_queue(&vsock->dev, &vsock->send_pkt_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) mutex_unlock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) err_vq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) vhost_vq_set_backend(vq, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) mutex_unlock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) vq = &vsock->vqs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) mutex_lock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) vhost_vq_set_backend(vq, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) mutex_unlock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) mutex_unlock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) size_t i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) mutex_lock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (check_owner) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) ret = vhost_dev_check_owner(&vsock->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) struct vhost_virtqueue *vq = &vsock->vqs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) mutex_lock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) vhost_vq_set_backend(vq, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) mutex_unlock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) mutex_unlock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) static void vhost_vsock_free(struct vhost_vsock *vsock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) kvfree(vsock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) struct vhost_virtqueue **vqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) struct vhost_vsock *vsock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) /* This struct is large and allocation could fail, fall back to vmalloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) * if there is no other way.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) vsock = kvmalloc(sizeof(*vsock), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) if (!vsock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) vqs = kmalloc_array(ARRAY_SIZE(vsock->vqs), sizeof(*vqs), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) if (!vqs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) vsock->guest_cid = 0; /* no CID assigned yet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) atomic_set(&vsock->queued_replies, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) VHOST_VSOCK_WEIGHT, true, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) file->private_data = vsock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) spin_lock_init(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) INIT_LIST_HEAD(&vsock->send_pkt_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) vhost_vsock_free(vsock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) static void vhost_vsock_flush(struct vhost_vsock *vsock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) if (vsock->vqs[i].handle_kick)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) vhost_poll_flush(&vsock->vqs[i].poll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) vhost_work_flush(&vsock->dev, &vsock->send_pkt_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) static void vhost_vsock_reset_orphans(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) struct vsock_sock *vsk = vsock_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) /* vmci_transport.c doesn't take sk_lock here either. At least we're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * under vsock_table_lock so the sock cannot disappear while we're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) * executing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) /* If the peer is still valid, no need to reset connection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) if (vhost_vsock_get(vsk->remote_addr.svm_cid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) /* If the close timeout is pending, let it expire. This avoids races
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * with the timeout callback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) if (vsk->close_work_scheduled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) sock_set_flag(sk, SOCK_DONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) vsk->peer_shutdown = SHUTDOWN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) sk->sk_state = SS_UNCONNECTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) sk->sk_err = ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) sk->sk_error_report(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) static int vhost_vsock_dev_release(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) struct vhost_vsock *vsock = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) mutex_lock(&vhost_vsock_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) if (vsock->guest_cid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) hash_del_rcu(&vsock->hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) mutex_unlock(&vhost_vsock_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) /* Wait for other CPUs to finish using vsock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) /* Iterating over all connections for all CIDs to find orphans is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * inefficient. Room for improvement here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) vsock_for_each_connected_socket(vhost_vsock_reset_orphans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) /* Don't check the owner, because we are in the release path, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * need to stop the vsock device in any case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * vhost_vsock_stop() can not fail in this case, so we don't need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * check the return code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) vhost_vsock_stop(vsock, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) vhost_vsock_flush(vsock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) vhost_dev_stop(&vsock->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) spin_lock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) while (!list_empty(&vsock->send_pkt_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) struct virtio_vsock_pkt *pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) pkt = list_first_entry(&vsock->send_pkt_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) struct virtio_vsock_pkt, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) list_del_init(&pkt->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) virtio_transport_free_pkt(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) spin_unlock_bh(&vsock->send_pkt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) vhost_dev_cleanup(&vsock->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) kfree(vsock->dev.vqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) vhost_vsock_free(vsock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) struct vhost_vsock *other;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) /* Refuse reserved CIDs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) if (guest_cid <= VMADDR_CID_HOST ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) guest_cid == U32_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) /* 64-bit CIDs are not yet supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) if (guest_cid > U32_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) /* Refuse if CID is assigned to the guest->host transport (i.e. nested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * VM), to make the loopback work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) if (vsock_find_cid(guest_cid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) return -EADDRINUSE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) /* Refuse if CID is already in use */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) mutex_lock(&vhost_vsock_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) other = vhost_vsock_get(guest_cid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) if (other && other != vsock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) mutex_unlock(&vhost_vsock_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) return -EADDRINUSE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (vsock->guest_cid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) hash_del_rcu(&vsock->hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) vsock->guest_cid = guest_cid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) mutex_unlock(&vhost_vsock_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) struct vhost_virtqueue *vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) if (features & ~VHOST_VSOCK_FEATURES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) mutex_lock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) if ((features & (1 << VHOST_F_LOG_ALL)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) !vhost_log_access_ok(&vsock->dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) mutex_unlock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) vq = &vsock->vqs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) mutex_lock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) vq->acked_features = features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) mutex_unlock(&vq->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) mutex_unlock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) struct vhost_vsock *vsock = f->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) void __user *argp = (void __user *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) u64 guest_cid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) u64 features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) int start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) switch (ioctl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) case VHOST_VSOCK_SET_GUEST_CID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) if (copy_from_user(&guest_cid, argp, sizeof(guest_cid)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) return vhost_vsock_set_cid(vsock, guest_cid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) case VHOST_VSOCK_SET_RUNNING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) if (copy_from_user(&start, argp, sizeof(start)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) if (start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) return vhost_vsock_start(vsock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) return vhost_vsock_stop(vsock, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) case VHOST_GET_FEATURES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) features = VHOST_VSOCK_FEATURES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) if (copy_to_user(argp, &features, sizeof(features)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) case VHOST_SET_FEATURES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (copy_from_user(&features, argp, sizeof(features)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) return vhost_vsock_set_features(vsock, features);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) mutex_lock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) if (r == -ENOIOCTLCMD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) r = vhost_vring_ioctl(&vsock->dev, ioctl, argp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) vhost_vsock_flush(vsock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) mutex_unlock(&vsock->dev.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) static const struct file_operations vhost_vsock_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) .open = vhost_vsock_dev_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) .release = vhost_vsock_dev_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) .llseek = noop_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) .unlocked_ioctl = vhost_vsock_dev_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) .compat_ioctl = compat_ptr_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) static struct miscdevice vhost_vsock_misc = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) .minor = VHOST_VSOCK_MINOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) .name = "vhost-vsock",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) .fops = &vhost_vsock_fops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) static int __init vhost_vsock_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) ret = vsock_core_register(&vhost_transport.transport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) VSOCK_TRANSPORT_F_H2G);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) return misc_register(&vhost_vsock_misc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) static void __exit vhost_vsock_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) misc_deregister(&vhost_vsock_misc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) vsock_core_unregister(&vhost_transport.transport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) module_init(vhost_vsock_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) module_exit(vhost_vsock_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) MODULE_AUTHOR("Asias He");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) MODULE_DESCRIPTION("vhost transport for vsock ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) MODULE_ALIAS_MISCDEV(VHOST_VSOCK_MINOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) MODULE_ALIAS("devname:vhost-vsock");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags