^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * (c) 2017 Stefano Stabellini <stefano@aporeto.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/net.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/socket.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <net/sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <xen/events.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <xen/grant_table.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <xen/xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <xen/xenbus.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <xen/interface/io/pvcalls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "pvcalls-front.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #define PVCALLS_INVALID_ID UINT_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define PVCALLS_RING_ORDER XENBUS_MAX_RING_GRANT_ORDER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define PVCALLS_NR_RSP_PER_RING __CONST_RING_SIZE(xen_pvcalls, XEN_PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define PVCALLS_FRONT_MAX_SPIN 5000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) static struct proto pvcalls_proto = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) .name = "PVCalls",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) .obj_size = sizeof(struct sock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct pvcalls_bedata {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) struct xen_pvcalls_front_ring ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) grant_ref_t ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) struct list_head socket_mappings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) spinlock_t socket_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) wait_queue_head_t inflight_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct xen_pvcalls_response rsp[PVCALLS_NR_RSP_PER_RING];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* Only one front/back connection supported. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static struct xenbus_device *pvcalls_front_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static atomic_t pvcalls_refcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /* first increment refcount, then proceed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define pvcalls_enter() { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) atomic_inc(&pvcalls_refcount); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /* first complete other operations, then decrement refcount */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define pvcalls_exit() { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) atomic_dec(&pvcalls_refcount); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct sock_mapping {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) bool active_socket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) struct socket *sock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) atomic_t refcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) grant_ref_t ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct pvcalls_data_intf *ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct pvcalls_data data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct mutex in_mutex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct mutex out_mutex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) wait_queue_head_t inflight_conn_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) } active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * Socket status, needs to be 64-bit aligned due to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * test_and_* functions which have this requirement on arm64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define PVCALLS_STATUS_UNINITALIZED 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define PVCALLS_STATUS_BIND 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #define PVCALLS_STATUS_LISTEN 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) uint8_t status __attribute__((aligned(8)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * Internal state-machine flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * Only one accept operation can be inflight for a socket.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * Only one poll operation can be inflight for a given socket.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * flags needs to be 64-bit aligned due to the test_and_*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * functions which have this requirement on arm64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #define PVCALLS_FLAG_ACCEPT_INFLIGHT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #define PVCALLS_FLAG_POLL_INFLIGHT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define PVCALLS_FLAG_POLL_RET 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) uint8_t flags __attribute__((aligned(8)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) uint32_t inflight_req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) struct sock_mapping *accept_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) wait_queue_head_t inflight_accept_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) } passive;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static inline struct sock_mapping *pvcalls_enter_sock(struct socket *sock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct sock_mapping *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (!pvcalls_front_dev ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) dev_get_drvdata(&pvcalls_front_dev->dev) == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return ERR_PTR(-ENOTCONN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) map = (struct sock_mapping *)sock->sk->sk_send_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (map == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) return ERR_PTR(-ENOTSOCK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) pvcalls_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) atomic_inc(&map->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static inline void pvcalls_exit_sock(struct socket *sock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct sock_mapping *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) map = (struct sock_mapping *)sock->sk->sk_send_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) atomic_dec(&map->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) pvcalls_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) static inline int get_request(struct pvcalls_bedata *bedata, int *req_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) *req_id = bedata->ring.req_prod_pvt & (RING_SIZE(&bedata->ring) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (RING_FULL(&bedata->ring) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) bedata->rsp[*req_id].req_id != PVCALLS_INVALID_ID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) static bool pvcalls_front_write_todo(struct sock_mapping *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct pvcalls_data_intf *intf = map->active.ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) RING_IDX cons, prod, size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) int32_t error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) error = intf->out_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) if (error == -ENOTCONN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) if (error != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) cons = intf->out_cons;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) prod = intf->out_prod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) return !!(size - pvcalls_queued(prod, cons, size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static bool pvcalls_front_read_todo(struct sock_mapping *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct pvcalls_data_intf *intf = map->active.ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) RING_IDX cons, prod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) int32_t error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) cons = intf->in_cons;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) prod = intf->in_prod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) error = intf->in_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) return (error != 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) pvcalls_queued(prod, cons,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER)) != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static irqreturn_t pvcalls_front_event_handler(int irq, void *dev_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct xenbus_device *dev = dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct pvcalls_bedata *bedata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) struct xen_pvcalls_response *rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) uint8_t *src, *dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) int req_id = 0, more = 0, done = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (dev == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) pvcalls_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) bedata = dev_get_drvdata(&dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (bedata == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) pvcalls_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) while (RING_HAS_UNCONSUMED_RESPONSES(&bedata->ring)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) rsp = RING_GET_RESPONSE(&bedata->ring, bedata->ring.rsp_cons);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) req_id = rsp->req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) if (rsp->cmd == PVCALLS_POLL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct sock_mapping *map = (struct sock_mapping *)(uintptr_t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) rsp->u.poll.id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) clear_bit(PVCALLS_FLAG_POLL_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) (void *)&map->passive.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * clear INFLIGHT, then set RET. It pairs with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * the checks at the beginning of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * pvcalls_front_poll_passive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) set_bit(PVCALLS_FLAG_POLL_RET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) (void *)&map->passive.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) dst = (uint8_t *)&bedata->rsp[req_id] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) sizeof(rsp->req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) src = (uint8_t *)rsp + sizeof(rsp->req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) memcpy(dst, src, sizeof(*rsp) - sizeof(rsp->req_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * First copy the rest of the data, then req_id. It is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * paired with the barrier when accessing bedata->rsp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) bedata->rsp[req_id].req_id = req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) done = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) bedata->ring.rsp_cons++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) RING_FINAL_CHECK_FOR_RESPONSES(&bedata->ring, more);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (more)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) wake_up(&bedata->inflight_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) pvcalls_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) static void pvcalls_front_free_map(struct pvcalls_bedata *bedata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) struct sock_mapping *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) unbind_from_irqhandler(map->active.irq, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) spin_lock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (!list_empty(&map->list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) list_del_init(&map->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) for (i = 0; i < (1 << PVCALLS_RING_ORDER); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) gnttab_end_foreign_access(map->active.ring->ref[i], 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) gnttab_end_foreign_access(map->active.ref, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) free_page((unsigned long)map->active.ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) kfree(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) static irqreturn_t pvcalls_front_conn_handler(int irq, void *sock_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct sock_mapping *map = sock_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) if (map == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) wake_up_interruptible(&map->active.inflight_conn_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) int pvcalls_front_socket(struct socket *sock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) struct pvcalls_bedata *bedata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) struct sock_mapping *map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) struct xen_pvcalls_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) int notify, req_id, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * PVCalls only supports domain AF_INET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * type SOCK_STREAM and protocol 0 sockets for now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * Check socket type here, AF_INET and protocol checks are done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * by the caller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (sock->type != SOCK_STREAM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) pvcalls_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (!pvcalls_front_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) pvcalls_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return -EACCES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) map = kzalloc(sizeof(*map), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (map == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) pvcalls_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) spin_lock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) ret = get_request(bedata, &req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) kfree(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) pvcalls_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * sock->sk->sk_send_head is not used for ip sockets: reuse the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * field to store a pointer to the struct sock_mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * corresponding to the socket. This way, we can easily get the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * struct sock_mapping from the struct socket.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) sock->sk->sk_send_head = (void *)map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) list_add_tail(&map->list, &bedata->socket_mappings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) req = RING_GET_REQUEST(&bedata->ring, req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) req->req_id = req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) req->cmd = PVCALLS_SOCKET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) req->u.socket.id = (uintptr_t) map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) req->u.socket.domain = AF_INET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) req->u.socket.type = SOCK_STREAM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) req->u.socket.protocol = IPPROTO_IP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) bedata->ring.req_prod_pvt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (notify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) notify_remote_via_irq(bedata->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) wait_event(bedata->inflight_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /* read req_id, then the content */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) ret = bedata->rsp[req_id].ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) pvcalls_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static void free_active_ring(struct sock_mapping *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) if (!map->active.ring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) free_pages_exact(map->active.data.in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) PAGE_SIZE << map->active.ring->ring_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) free_page((unsigned long)map->active.ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) static int alloc_active_ring(struct sock_mapping *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) void *bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) map->active.ring = (struct pvcalls_data_intf *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) get_zeroed_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (!map->active.ring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) map->active.ring->ring_order = PVCALLS_RING_ORDER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) bytes = alloc_pages_exact(PAGE_SIZE << PVCALLS_RING_ORDER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) GFP_KERNEL | __GFP_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (!bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) map->active.data.in = bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) map->active.data.out = bytes +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) free_active_ring(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) static int create_active(struct sock_mapping *map, evtchn_port_t *evtchn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) void *bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) int ret, irq = -1, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) *evtchn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) init_waitqueue_head(&map->active.inflight_conn_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) bytes = map->active.data.in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) for (i = 0; i < (1 << PVCALLS_RING_ORDER); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) map->active.ring->ref[i] = gnttab_grant_foreign_access(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) pvcalls_front_dev->otherend_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) pfn_to_gfn(virt_to_pfn(bytes) + i), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) map->active.ref = gnttab_grant_foreign_access(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) pvcalls_front_dev->otherend_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) pfn_to_gfn(virt_to_pfn((void *)map->active.ring)), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) ret = xenbus_alloc_evtchn(pvcalls_front_dev, evtchn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) goto out_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) irq = bind_evtchn_to_irqhandler(*evtchn, pvcalls_front_conn_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 0, "pvcalls-frontend", map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) if (irq < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) ret = irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) goto out_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) map->active.irq = irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) map->active_socket = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) mutex_init(&map->active.in_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) mutex_init(&map->active.out_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) out_error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (*evtchn > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) xenbus_free_evtchn(pvcalls_front_dev, *evtchn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) int pvcalls_front_connect(struct socket *sock, struct sockaddr *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) int addr_len, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct pvcalls_bedata *bedata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) struct sock_mapping *map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) struct xen_pvcalls_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) int notify, req_id, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) evtchn_port_t evtchn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (addr->sa_family != AF_INET || sock->type != SOCK_STREAM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) map = pvcalls_enter_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (IS_ERR(map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) return PTR_ERR(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) ret = alloc_active_ring(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) spin_lock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) ret = get_request(bedata, &req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) free_active_ring(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) ret = create_active(map, &evtchn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) free_active_ring(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) req = RING_GET_REQUEST(&bedata->ring, req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) req->req_id = req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) req->cmd = PVCALLS_CONNECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) req->u.connect.id = (uintptr_t)map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) req->u.connect.len = addr_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) req->u.connect.flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) req->u.connect.ref = map->active.ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) req->u.connect.evtchn = evtchn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) memcpy(req->u.connect.addr, addr, sizeof(*addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) map->sock = sock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) bedata->ring.req_prod_pvt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (notify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) notify_remote_via_irq(bedata->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) wait_event(bedata->inflight_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) /* read req_id, then the content */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) ret = bedata->rsp[req_id].ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) static int __write_ring(struct pvcalls_data_intf *intf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) struct pvcalls_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) struct iov_iter *msg_iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) RING_IDX cons, prod, size, masked_prod, masked_cons;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) RING_IDX array_size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) int32_t error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) error = intf->out_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (error < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) cons = intf->out_cons;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) prod = intf->out_prod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) /* read indexes before continuing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) virt_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) size = pvcalls_queued(prod, cons, array_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (size > array_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (size == array_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (len > array_size - size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) len = array_size - size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) masked_prod = pvcalls_mask(prod, array_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) masked_cons = pvcalls_mask(cons, array_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (masked_prod < masked_cons) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) len = copy_from_iter(data->out + masked_prod, len, msg_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (len > array_size - masked_prod) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) int ret = copy_from_iter(data->out + masked_prod,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) array_size - masked_prod, msg_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) if (ret != array_size - masked_prod) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) len = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) len = ret + copy_from_iter(data->out, len - ret, msg_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) len = copy_from_iter(data->out + masked_prod, len, msg_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) /* write to ring before updating pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) virt_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) intf->out_prod += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) int pvcalls_front_sendmsg(struct socket *sock, struct msghdr *msg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct sock_mapping *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) int sent, tot_sent = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) int count = 0, flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) flags = msg->msg_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (flags & (MSG_CONFIRM|MSG_DONTROUTE|MSG_EOR|MSG_OOB))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) map = pvcalls_enter_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) if (IS_ERR(map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) return PTR_ERR(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) mutex_lock(&map->active.out_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if ((flags & MSG_DONTWAIT) && !pvcalls_front_write_todo(map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) mutex_unlock(&map->active.out_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) if (len > INT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) len = INT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) sent = __write_ring(map->active.ring,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) &map->active.data, &msg->msg_iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (sent > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) len -= sent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) tot_sent += sent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) notify_remote_via_irq(map->active.irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (sent >= 0 && len > 0 && count < PVCALLS_FRONT_MAX_SPIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) if (sent < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) tot_sent = sent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) mutex_unlock(&map->active.out_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) return tot_sent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) static int __read_ring(struct pvcalls_data_intf *intf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) struct pvcalls_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) struct iov_iter *msg_iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) size_t len, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) RING_IDX cons, prod, size, masked_prod, masked_cons;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) RING_IDX array_size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) int32_t error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) cons = intf->in_cons;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) prod = intf->in_prod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) error = intf->in_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) /* get pointers before reading from the ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) virt_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) size = pvcalls_queued(prod, cons, array_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) masked_prod = pvcalls_mask(prod, array_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) masked_cons = pvcalls_mask(cons, array_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) if (size == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) return error ?: size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (len > size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) len = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (masked_prod > masked_cons) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) len = copy_to_iter(data->in + masked_cons, len, msg_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) if (len > (array_size - masked_cons)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) int ret = copy_to_iter(data->in + masked_cons,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) array_size - masked_cons, msg_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) if (ret != array_size - masked_cons) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) len = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) len = ret + copy_to_iter(data->in, len - ret, msg_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) len = copy_to_iter(data->in + masked_cons, len, msg_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) /* read data from the ring before increasing the index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) virt_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (!(flags & MSG_PEEK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) intf->in_cons += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) int pvcalls_front_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) struct sock_mapping *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (flags & (MSG_CMSG_CLOEXEC|MSG_ERRQUEUE|MSG_OOB|MSG_TRUNC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) map = pvcalls_enter_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) if (IS_ERR(map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) return PTR_ERR(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) mutex_lock(&map->active.in_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (len > XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) len = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) while (!(flags & MSG_DONTWAIT) && !pvcalls_front_read_todo(map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) wait_event_interruptible(map->active.inflight_conn_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) pvcalls_front_read_todo(map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) ret = __read_ring(map->active.ring, &map->active.data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) &msg->msg_iter, len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) notify_remote_via_irq(map->active.irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) ret = (flags & MSG_DONTWAIT) ? -EAGAIN : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) if (ret == -ENOTCONN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) mutex_unlock(&map->active.in_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) int pvcalls_front_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) struct pvcalls_bedata *bedata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) struct sock_mapping *map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) struct xen_pvcalls_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) int notify, req_id, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) if (addr->sa_family != AF_INET || sock->type != SOCK_STREAM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) map = pvcalls_enter_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) if (IS_ERR(map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) return PTR_ERR(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) spin_lock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) ret = get_request(bedata, &req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) req = RING_GET_REQUEST(&bedata->ring, req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) req->req_id = req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) map->sock = sock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) req->cmd = PVCALLS_BIND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) req->u.bind.id = (uintptr_t)map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) memcpy(req->u.bind.addr, addr, sizeof(*addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) req->u.bind.len = addr_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) init_waitqueue_head(&map->passive.inflight_accept_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) map->active_socket = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) bedata->ring.req_prod_pvt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) if (notify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) notify_remote_via_irq(bedata->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) wait_event(bedata->inflight_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) /* read req_id, then the content */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) ret = bedata->rsp[req_id].ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) map->passive.status = PVCALLS_STATUS_BIND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) int pvcalls_front_listen(struct socket *sock, int backlog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) struct pvcalls_bedata *bedata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) struct sock_mapping *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) struct xen_pvcalls_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) int notify, req_id, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) map = pvcalls_enter_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) if (IS_ERR(map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) return PTR_ERR(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (map->passive.status != PVCALLS_STATUS_BIND) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) spin_lock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) ret = get_request(bedata, &req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) req = RING_GET_REQUEST(&bedata->ring, req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) req->req_id = req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) req->cmd = PVCALLS_LISTEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) req->u.listen.id = (uintptr_t) map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) req->u.listen.backlog = backlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) bedata->ring.req_prod_pvt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (notify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) notify_remote_via_irq(bedata->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) wait_event(bedata->inflight_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) /* read req_id, then the content */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) ret = bedata->rsp[req_id].ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) map->passive.status = PVCALLS_STATUS_LISTEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) int pvcalls_front_accept(struct socket *sock, struct socket *newsock, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) struct pvcalls_bedata *bedata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) struct sock_mapping *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) struct sock_mapping *map2 = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) struct xen_pvcalls_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) int notify, req_id, ret, nonblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) evtchn_port_t evtchn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) map = pvcalls_enter_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (IS_ERR(map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) return PTR_ERR(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) if (map->passive.status != PVCALLS_STATUS_LISTEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) nonblock = flags & SOCK_NONBLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * Backend only supports 1 inflight accept request, will return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) * errors for the others
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) if (test_and_set_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) (void *)&map->passive.flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) req_id = READ_ONCE(map->passive.inflight_req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if (req_id != PVCALLS_INVALID_ID &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) READ_ONCE(bedata->rsp[req_id].req_id) == req_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) map2 = map->passive.accept_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) goto received;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) if (nonblock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) if (wait_event_interruptible(map->passive.inflight_accept_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) !test_and_set_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) (void *)&map->passive.flags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) map2 = kzalloc(sizeof(*map2), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) if (map2 == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) (void *)&map->passive.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) ret = alloc_active_ring(map2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) (void *)&map->passive.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) kfree(map2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) spin_lock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) ret = get_request(bedata, &req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) (void *)&map->passive.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) free_active_ring(map2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) kfree(map2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) ret = create_active(map2, &evtchn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) free_active_ring(map2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) kfree(map2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) (void *)&map->passive.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) list_add_tail(&map2->list, &bedata->socket_mappings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) req = RING_GET_REQUEST(&bedata->ring, req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) req->req_id = req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) req->cmd = PVCALLS_ACCEPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) req->u.accept.id = (uintptr_t) map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) req->u.accept.ref = map2->active.ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) req->u.accept.id_new = (uintptr_t) map2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) req->u.accept.evtchn = evtchn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) map->passive.accept_map = map2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) bedata->ring.req_prod_pvt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (notify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) notify_remote_via_irq(bedata->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) /* We could check if we have received a response before returning. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) if (nonblock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) WRITE_ONCE(map->passive.inflight_req_id, req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) if (wait_event_interruptible(bedata->inflight_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) READ_ONCE(bedata->rsp[req_id].req_id) == req_id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) /* read req_id, then the content */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) received:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) map2->sock = newsock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) newsock->sk = sk_alloc(sock_net(sock->sk), PF_INET, GFP_KERNEL, &pvcalls_proto, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) if (!newsock->sk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) map->passive.inflight_req_id = PVCALLS_INVALID_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) (void *)&map->passive.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) pvcalls_front_free_map(bedata, map2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) newsock->sk->sk_send_head = (void *)map2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) ret = bedata->rsp[req_id].ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) map->passive.inflight_req_id = PVCALLS_INVALID_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT, (void *)&map->passive.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) wake_up(&map->passive.inflight_accept_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) static __poll_t pvcalls_front_poll_passive(struct file *file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) struct pvcalls_bedata *bedata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) struct sock_mapping *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) poll_table *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) int notify, req_id, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) struct xen_pvcalls_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (test_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) (void *)&map->passive.flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) uint32_t req_id = READ_ONCE(map->passive.inflight_req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) if (req_id != PVCALLS_INVALID_ID &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) READ_ONCE(bedata->rsp[req_id].req_id) == req_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) return EPOLLIN | EPOLLRDNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) poll_wait(file, &map->passive.inflight_accept_req, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (test_and_clear_bit(PVCALLS_FLAG_POLL_RET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) (void *)&map->passive.flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) return EPOLLIN | EPOLLRDNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) * First check RET, then INFLIGHT. No barriers necessary to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) * ensure execution ordering because of the conditional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) * instructions creating control dependencies.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) if (test_and_set_bit(PVCALLS_FLAG_POLL_INFLIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) (void *)&map->passive.flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) poll_wait(file, &bedata->inflight_req, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) spin_lock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) ret = get_request(bedata, &req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) req = RING_GET_REQUEST(&bedata->ring, req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) req->req_id = req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) req->cmd = PVCALLS_POLL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) req->u.poll.id = (uintptr_t) map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) bedata->ring.req_prod_pvt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) if (notify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) notify_remote_via_irq(bedata->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) poll_wait(file, &bedata->inflight_req, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) static __poll_t pvcalls_front_poll_active(struct file *file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) struct pvcalls_bedata *bedata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) struct sock_mapping *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) poll_table *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) __poll_t mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) int32_t in_error, out_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) struct pvcalls_data_intf *intf = map->active.ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) out_error = intf->out_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) in_error = intf->in_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) poll_wait(file, &map->active.inflight_conn_req, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) if (pvcalls_front_write_todo(map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) mask |= EPOLLOUT | EPOLLWRNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (pvcalls_front_read_todo(map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) mask |= EPOLLIN | EPOLLRDNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) if (in_error != 0 || out_error != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) mask |= EPOLLERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) return mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) __poll_t pvcalls_front_poll(struct file *file, struct socket *sock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) poll_table *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) struct pvcalls_bedata *bedata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) struct sock_mapping *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) __poll_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) map = pvcalls_enter_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) if (IS_ERR(map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) return EPOLLNVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) if (map->active_socket)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) ret = pvcalls_front_poll_active(file, bedata, map, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) ret = pvcalls_front_poll_passive(file, bedata, map, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) int pvcalls_front_release(struct socket *sock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) struct pvcalls_bedata *bedata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) struct sock_mapping *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) int req_id, notify, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) struct xen_pvcalls_request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) if (sock->sk == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) map = pvcalls_enter_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) if (IS_ERR(map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) if (PTR_ERR(map) == -ENOTCONN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) spin_lock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) ret = get_request(bedata, &req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) pvcalls_exit_sock(sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) sock->sk->sk_send_head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) req = RING_GET_REQUEST(&bedata->ring, req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) req->req_id = req_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) req->cmd = PVCALLS_RELEASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) req->u.release.id = (uintptr_t)map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) bedata->ring.req_prod_pvt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) if (notify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) notify_remote_via_irq(bedata->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) wait_event(bedata->inflight_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) if (map->active_socket) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) * Set in_error and wake up inflight_conn_req to force
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) * recvmsg waiters to exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) map->active.ring->in_error = -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) wake_up_interruptible(&map->active.inflight_conn_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) * We need to make sure that sendmsg/recvmsg on this socket have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) * not started before we've cleared sk_send_head here. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) * easiest way to guarantee this is to see that no pvcalls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) * (other than us) is in progress on this socket.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) while (atomic_read(&map->refcount) > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) pvcalls_front_free_map(bedata, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) wake_up(&bedata->inflight_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) wake_up(&map->passive.inflight_accept_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) while (atomic_read(&map->refcount) > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) spin_lock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) list_del(&map->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) spin_unlock(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) if (READ_ONCE(map->passive.inflight_req_id) != PVCALLS_INVALID_ID &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) READ_ONCE(map->passive.inflight_req_id) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) pvcalls_front_free_map(bedata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) map->passive.accept_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) kfree(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) WRITE_ONCE(bedata->rsp[req_id].req_id, PVCALLS_INVALID_ID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) pvcalls_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) static const struct xenbus_device_id pvcalls_front_ids[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) { "pvcalls" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) { "" }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) static int pvcalls_front_remove(struct xenbus_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) struct pvcalls_bedata *bedata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) struct sock_mapping *map = NULL, *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) dev_set_drvdata(&dev->dev, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) pvcalls_front_dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) if (bedata->irq >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) unbind_from_irqhandler(bedata->irq, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) map->sock->sk->sk_send_head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) if (map->active_socket) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) map->active.ring->in_error = -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) wake_up_interruptible(&map->active.inflight_conn_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) while (atomic_read(&pvcalls_refcount) > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) if (map->active_socket) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) /* No need to lock, refcount is 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) pvcalls_front_free_map(bedata, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) list_del(&map->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) kfree(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) if (bedata->ref != -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) gnttab_end_foreign_access(bedata->ref, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) kfree(bedata->ring.sring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) kfree(bedata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) xenbus_switch_state(dev, XenbusStateClosed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) static int pvcalls_front_probe(struct xenbus_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) const struct xenbus_device_id *id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) int ret = -ENOMEM, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) evtchn_port_t evtchn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) unsigned int max_page_order, function_calls, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) char *versions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) grant_ref_t gref_head = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) struct xenbus_transaction xbt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) struct pvcalls_bedata *bedata = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) struct xen_pvcalls_sring *sring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) if (pvcalls_front_dev != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) dev_err(&dev->dev, "only one PV Calls connection supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) versions = xenbus_read(XBT_NIL, dev->otherend, "versions", &len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) if (IS_ERR(versions))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) return PTR_ERR(versions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) if (!len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) if (strcmp(versions, "1")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) kfree(versions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) kfree(versions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) max_page_order = xenbus_read_unsigned(dev->otherend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) "max-page-order", 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) if (max_page_order < PVCALLS_RING_ORDER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) function_calls = xenbus_read_unsigned(dev->otherend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) "function-calls", 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) /* See XENBUS_FUNCTIONS_CALLS in pvcalls.h */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) if (function_calls != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) pr_info("%s max-page-order is %u\n", __func__, max_page_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) bedata = kzalloc(sizeof(struct pvcalls_bedata), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) if (!bedata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) dev_set_drvdata(&dev->dev, bedata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) pvcalls_front_dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) init_waitqueue_head(&bedata->inflight_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) INIT_LIST_HEAD(&bedata->socket_mappings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) spin_lock_init(&bedata->socket_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) bedata->irq = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) bedata->ref = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) for (i = 0; i < PVCALLS_NR_RSP_PER_RING; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) bedata->rsp[i].req_id = PVCALLS_INVALID_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) sring = (struct xen_pvcalls_sring *) __get_free_page(GFP_KERNEL |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) __GFP_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) if (!sring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) SHARED_RING_INIT(sring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) FRONT_RING_INIT(&bedata->ring, sring, XEN_PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) ret = xenbus_alloc_evtchn(dev, &evtchn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) bedata->irq = bind_evtchn_to_irqhandler(evtchn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) pvcalls_front_event_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) 0, "pvcalls-frontend", dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (bedata->irq < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) ret = bedata->irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) ret = gnttab_alloc_grant_references(1, &gref_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) ret = gnttab_claim_grant_reference(&gref_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) bedata->ref = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) gnttab_grant_foreign_access_ref(bedata->ref, dev->otherend_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) virt_to_gfn((void *)sring), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) ret = xenbus_transaction_start(&xbt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) xenbus_dev_fatal(dev, ret, "starting transaction");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) ret = xenbus_printf(xbt, dev->nodename, "version", "%u", 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) goto error_xenbus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) ret = xenbus_printf(xbt, dev->nodename, "ring-ref", "%d", bedata->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) goto error_xenbus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) ret = xenbus_printf(xbt, dev->nodename, "port", "%u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) evtchn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) goto error_xenbus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) ret = xenbus_transaction_end(xbt, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) if (ret == -EAGAIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) xenbus_dev_fatal(dev, ret, "completing transaction");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) xenbus_switch_state(dev, XenbusStateInitialised);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) error_xenbus:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) xenbus_transaction_end(xbt, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) xenbus_dev_fatal(dev, ret, "writing xenstore");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) pvcalls_front_remove(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) static void pvcalls_front_changed(struct xenbus_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) enum xenbus_state backend_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) switch (backend_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) case XenbusStateReconfiguring:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) case XenbusStateReconfigured:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) case XenbusStateInitialising:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) case XenbusStateInitialised:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) case XenbusStateUnknown:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) case XenbusStateInitWait:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) case XenbusStateConnected:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) xenbus_switch_state(dev, XenbusStateConnected);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) case XenbusStateClosed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) if (dev->state == XenbusStateClosed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) /* Missed the backend's CLOSING state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) case XenbusStateClosing:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) xenbus_frontend_closed(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) static struct xenbus_driver pvcalls_front_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) .ids = pvcalls_front_ids,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) .probe = pvcalls_front_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) .remove = pvcalls_front_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) .otherend_changed = pvcalls_front_changed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) static int __init pvcalls_frontend_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) if (!xen_domain())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) pr_info("Initialising Xen pvcalls frontend driver\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) return xenbus_register_frontend(&pvcalls_front_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) module_init(pvcalls_frontend_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) MODULE_DESCRIPTION("Xen PV Calls frontend driver");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) MODULE_AUTHOR("Stefano Stabellini <sstabellini@kernel.org>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) MODULE_LICENSE("GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags