^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * I/O Address Space ID allocator. There is one global IOASID space, split into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * subsets. Users create a subset with DECLARE_IOASID_SET, then allocate and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * free IOASIDs with ioasid_alloc and ioasid_free.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/ioasid.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/xarray.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) struct ioasid_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) ioasid_t id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) struct ioasid_set *set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) void *private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * struct ioasid_allocator_data - Internal data structure to hold information
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * about an allocator. There are two types of allocators:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * - Default allocator always has its own XArray to track the IOASIDs allocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * - Custom allocators may share allocation helpers with different private data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * Custom allocators that share the same helper functions also share the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * XArray.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * Rules:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * 1. Default allocator is always available, not dynamically registered. This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * to prevent race conditions with early boot code that want to register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * custom allocators or allocate IOASIDs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * 2. Custom allocators take precedence over the default allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * 3. When all custom allocators sharing the same helper functions are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * unregistered (e.g. due to hotplug), all outstanding IOASIDs must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * freed. Otherwise, outstanding IOASIDs will be lost and orphaned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * 4. When switching between custom allocators sharing the same helper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * functions, outstanding IOASIDs are preserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * 5. When switching between custom allocator and default allocator, all IOASIDs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * must be freed to ensure unadulterated space for the new allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * @ops: allocator helper functions and its data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * @list: registered custom allocators
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * @slist: allocators share the same ops but different data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * @flags: attributes of the allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * @xa: xarray holds the IOASID space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * @rcu: used for kfree_rcu when unregistering allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct ioasid_allocator_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct ioasid_allocator_ops *ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct list_head slist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define IOASID_ALLOCATOR_CUSTOM BIT(0) /* Needs framework to track results */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct xarray xa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static DEFINE_SPINLOCK(ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static LIST_HEAD(allocators_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static ioasid_t default_alloc(ioasid_t min, ioasid_t max, void *opaque);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) static void default_free(ioasid_t ioasid, void *opaque);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static struct ioasid_allocator_ops default_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) .alloc = default_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) .free = default_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) static struct ioasid_allocator_data default_allocator = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) .ops = &default_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) .flags = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) .xa = XARRAY_INIT(ioasid_xa, XA_FLAGS_ALLOC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static struct ioasid_allocator_data *active_allocator = &default_allocator;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static ioasid_t default_alloc(ioasid_t min, ioasid_t max, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) ioasid_t id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) if (xa_alloc(&default_allocator.xa, &id, opaque, XA_LIMIT(min, max), GFP_ATOMIC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) pr_err("Failed to alloc ioasid from %d to %d\n", min, max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) return INVALID_IOASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) return id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static void default_free(ioasid_t ioasid, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct ioasid_data *ioasid_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) ioasid_data = xa_erase(&default_allocator.xa, ioasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) kfree_rcu(ioasid_data, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) /* Allocate and initialize a new custom allocator with its helper functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static struct ioasid_allocator_data *ioasid_alloc_allocator(struct ioasid_allocator_ops *ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct ioasid_allocator_data *ia_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) ia_data = kzalloc(sizeof(*ia_data), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (!ia_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) xa_init_flags(&ia_data->xa, XA_FLAGS_ALLOC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) INIT_LIST_HEAD(&ia_data->slist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) ia_data->flags |= IOASID_ALLOCATOR_CUSTOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) ia_data->ops = ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /* For tracking custom allocators that share the same ops */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) list_add_tail(&ops->list, &ia_data->slist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return ia_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static bool use_same_ops(struct ioasid_allocator_ops *a, struct ioasid_allocator_ops *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return (a->free == b->free) && (a->alloc == b->alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * ioasid_register_allocator - register a custom allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * @ops: the custom allocator ops to be registered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * Custom allocators take precedence over the default xarray based allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * Private data associated with the IOASID allocated by the custom allocators
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * are managed by IOASID framework similar to data stored in xa by default
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * There can be multiple allocators registered but only one is active. In case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * of runtime removal of a custom allocator, the next one is activated based
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * on the registration ordering.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * Multiple allocators can share the same alloc() function, in this case the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * IOASID space is shared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) int ioasid_register_allocator(struct ioasid_allocator_ops *ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct ioasid_allocator_data *ia_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct ioasid_allocator_data *pallocator;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) spin_lock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) ia_data = ioasid_alloc_allocator(ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) if (!ia_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * No particular preference, we activate the first one and keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * the later registered allocators in a list in case the first one gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * removed due to hotplug.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (list_empty(&allocators_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) WARN_ON(active_allocator != &default_allocator);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* Use this new allocator if default is not active */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (xa_empty(&active_allocator->xa)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) rcu_assign_pointer(active_allocator, ia_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) list_add_tail(&ia_data->list, &allocators_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) pr_warn("Default allocator active with outstanding IOASID\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) /* Check if the allocator is already registered */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) list_for_each_entry(pallocator, &allocators_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (pallocator->ops == ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) pr_err("IOASID allocator already registered\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) ret = -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) } else if (use_same_ops(pallocator->ops, ops)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * If the new allocator shares the same ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * then they will share the same IOASID space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * We should put them under the same xarray.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) list_add_tail(&ops->list, &pallocator->slist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) list_add_tail(&ia_data->list, &allocators_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) spin_unlock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) kfree(ia_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) spin_unlock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) EXPORT_SYMBOL_GPL(ioasid_register_allocator);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * ioasid_unregister_allocator - Remove a custom IOASID allocator ops
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * @ops: the custom allocator to be removed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * Remove an allocator from the list, activate the next allocator in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * the order it was registered. Or revert to default allocator if all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * custom allocators are unregistered without outstanding IOASIDs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) void ioasid_unregister_allocator(struct ioasid_allocator_ops *ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) struct ioasid_allocator_data *pallocator;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) struct ioasid_allocator_ops *sops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) spin_lock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (list_empty(&allocators_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) pr_warn("No custom IOASID allocators active!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) goto exit_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) list_for_each_entry(pallocator, &allocators_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) if (!use_same_ops(pallocator->ops, ops))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (list_is_singular(&pallocator->slist)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /* No shared helper functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) list_del(&pallocator->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * All IOASIDs should have been freed before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * the last allocator that shares the same ops
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * is unregistered.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) WARN_ON(!xa_empty(&pallocator->xa));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (list_empty(&allocators_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) pr_info("No custom IOASID allocators, switch to default.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) rcu_assign_pointer(active_allocator, &default_allocator);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) } else if (pallocator == active_allocator) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) rcu_assign_pointer(active_allocator,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) list_first_entry(&allocators_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) struct ioasid_allocator_data, list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) pr_info("IOASID allocator changed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) kfree_rcu(pallocator, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * Find the matching shared ops to delete,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * but keep outstanding IOASIDs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) list_for_each_entry(sops, &pallocator->slist, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (sops == ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) list_del(&ops->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) exit_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) spin_unlock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) EXPORT_SYMBOL_GPL(ioasid_unregister_allocator);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * ioasid_set_data - Set private data for an allocated ioasid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * @ioasid: the ID to set data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * @data: the private data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * For IOASID that is already allocated, private data can be set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * via this API. Future lookup can be done via ioasid_find.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) int ioasid_set_data(ioasid_t ioasid, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct ioasid_data *ioasid_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) spin_lock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) ioasid_data = xa_load(&active_allocator->xa, ioasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (ioasid_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) rcu_assign_pointer(ioasid_data->private, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) ret = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) spin_unlock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * Wait for readers to stop accessing the old private data, so the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * caller can free it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) EXPORT_SYMBOL_GPL(ioasid_set_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * ioasid_alloc - Allocate an IOASID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * @set: the IOASID set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * @min: the minimum ID (inclusive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * @max: the maximum ID (inclusive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * @private: data private to the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * Allocate an ID between @min and @max. The @private pointer is stored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * internally and can be retrieved with ioasid_find().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * Return: the allocated ID on success, or %INVALID_IOASID on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, ioasid_t max,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) void *private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) struct ioasid_data *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) void *adata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) ioasid_t id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) data = kzalloc(sizeof(*data), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (!data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return INVALID_IOASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) data->set = set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) data->private = private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * Custom allocator needs allocator data to perform platform specific
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * operations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) spin_lock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) adata = active_allocator->flags & IOASID_ALLOCATOR_CUSTOM ? active_allocator->ops->pdata : data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) id = active_allocator->ops->alloc(min, max, adata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (id == INVALID_IOASID) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) pr_err("Failed ASID allocation %lu\n", active_allocator->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) goto exit_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if ((active_allocator->flags & IOASID_ALLOCATOR_CUSTOM) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) xa_alloc(&active_allocator->xa, &id, data, XA_LIMIT(id, id), GFP_ATOMIC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /* Custom allocator needs framework to store and track allocation results */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) pr_err("Failed to alloc ioasid from %d\n", id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) active_allocator->ops->free(id, active_allocator->ops->pdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) goto exit_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) data->id = id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) spin_unlock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) return id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) exit_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) spin_unlock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) kfree(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) return INVALID_IOASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) EXPORT_SYMBOL_GPL(ioasid_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * ioasid_free - Free an IOASID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * @ioasid: the ID to remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) void ioasid_free(ioasid_t ioasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) struct ioasid_data *ioasid_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) spin_lock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) ioasid_data = xa_load(&active_allocator->xa, ioasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (!ioasid_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) pr_err("Trying to free unknown IOASID %u\n", ioasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) goto exit_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) active_allocator->ops->free(ioasid, active_allocator->ops->pdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) /* Custom allocator needs additional steps to free the xa element */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) if (active_allocator->flags & IOASID_ALLOCATOR_CUSTOM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) ioasid_data = xa_erase(&active_allocator->xa, ioasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) kfree_rcu(ioasid_data, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) exit_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) spin_unlock(&ioasid_allocator_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) EXPORT_SYMBOL_GPL(ioasid_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * ioasid_find - Find IOASID data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * @set: the IOASID set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * @ioasid: the IOASID to find
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * @getter: function to call on the found object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * The optional getter function allows to take a reference to the found object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * under the rcu lock. The function can also check if the object is still valid:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * if @getter returns false, then the object is invalid and NULL is returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * If the IOASID exists, return the private pointer passed to ioasid_alloc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * Private data can be NULL if not set. Return an error if the IOASID is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * found, or if @set is not NULL and the IOASID does not belong to the set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) bool (*getter)(void *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) void *priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) struct ioasid_data *ioasid_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) struct ioasid_allocator_data *idata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) idata = rcu_dereference(active_allocator);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) ioasid_data = xa_load(&idata->xa, ioasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (!ioasid_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) priv = ERR_PTR(-ENOENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if (set && ioasid_data->set != set) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /* data found but does not belong to the set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) priv = ERR_PTR(-EACCES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /* Now IOASID and its set is verified, we can return the private data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) priv = rcu_dereference(ioasid_data->private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (getter && !getter(priv))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) priv = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) return priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) EXPORT_SYMBOL_GPL(ioasid_find);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) MODULE_AUTHOR("Jean-Philippe Brucker <jean-philippe.brucker@arm.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@linux.intel.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) MODULE_DESCRIPTION("IO Address Space ID (IOASID) allocator");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) MODULE_LICENSE("GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags