^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Copyright (C) 2020 Red Hat, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Author: Jason Wang <jasowang@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * IOTLB implementation for vhost.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/vhost_iotlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #define MOD_VERSION "0.1"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #define MOD_DESC "VHOST IOTLB"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #define MOD_LICENSE "GPL v2"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #define START(map) ((map)->start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define LAST(map) ((map)->last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) INTERVAL_TREE_DEFINE(struct vhost_iotlb_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) rb, __u64, __subtree_last,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) START, LAST, static inline, vhost_iotlb_itree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * vhost_iotlb_map_free - remove a map node and free it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * @iotlb: the IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * @map: the map that want to be remove and freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) struct vhost_iotlb_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) vhost_iotlb_itree_remove(map, &iotlb->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) list_del(&map->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) kfree(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) iotlb->nmaps--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) EXPORT_SYMBOL_GPL(vhost_iotlb_map_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * vhost_iotlb_add_range - add a new range to vhost IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * @iotlb: the IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * @start: start of the IOVA range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * @last: last of IOVA range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * @addr: the address that is mapped to @start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * @perm: access permission of this range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * Returns an error last is smaller than start or memory allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * fails
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) int vhost_iotlb_add_range(struct vhost_iotlb *iotlb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) u64 start, u64 last,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) u64 addr, unsigned int perm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct vhost_iotlb_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) if (last < start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) if (iotlb->limit &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) iotlb->nmaps == iotlb->limit &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) map = list_first_entry(&iotlb->list, typeof(*map), link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) vhost_iotlb_map_free(iotlb, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) map = kmalloc(sizeof(*map), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (!map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) map->start = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) map->size = last - start + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) map->last = last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) map->addr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) map->perm = perm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) iotlb->nmaps++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) vhost_iotlb_itree_insert(map, &iotlb->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) INIT_LIST_HEAD(&map->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) list_add_tail(&map->link, &iotlb->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) EXPORT_SYMBOL_GPL(vhost_iotlb_add_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * vring_iotlb_del_range - delete overlapped ranges from vhost IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * @iotlb: the IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * @start: start of the IOVA range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * @last: last of IOVA range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) struct vhost_iotlb_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) while ((map = vhost_iotlb_itree_iter_first(&iotlb->root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) start, last)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) vhost_iotlb_map_free(iotlb, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) EXPORT_SYMBOL_GPL(vhost_iotlb_del_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * vhost_iotlb_alloc - add a new vhost IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * @limit: maximum number of IOTLB entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * @flags: VHOST_IOTLB_FLAG_XXX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * Returns an error is memory allocation fails
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) struct vhost_iotlb *iotlb = kzalloc(sizeof(*iotlb), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (!iotlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) iotlb->root = RB_ROOT_CACHED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) iotlb->limit = limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) iotlb->nmaps = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) iotlb->flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) INIT_LIST_HEAD(&iotlb->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) return iotlb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) EXPORT_SYMBOL_GPL(vhost_iotlb_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * @iotlb: the IOTLB to be reset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) void vhost_iotlb_reset(struct vhost_iotlb *iotlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) EXPORT_SYMBOL_GPL(vhost_iotlb_reset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * vhost_iotlb_free - reset and free vhost IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * @iotlb: the IOTLB to be freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) void vhost_iotlb_free(struct vhost_iotlb *iotlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) if (iotlb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) vhost_iotlb_reset(iotlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) kfree(iotlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) EXPORT_SYMBOL_GPL(vhost_iotlb_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * vhost_iotlb_itree_first - return the first overlapped range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * @iotlb: the IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * @start: start of IOVA range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * @last: last byte in IOVA range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) struct vhost_iotlb_map *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) return vhost_iotlb_itree_iter_first(&iotlb->root, start, last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * vhost_iotlb_itree_next - return the next overlapped range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * @map: the starting map node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * @start: start of IOVA range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * @last: last byte IOVA range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct vhost_iotlb_map *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) return vhost_iotlb_itree_iter_next(map, start, last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) MODULE_VERSION(MOD_VERSION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) MODULE_DESCRIPTION(MOD_DESC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) MODULE_AUTHOR(MOD_AUTHOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) MODULE_LICENSE(MOD_LICENSE);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags