^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) //SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/bpf-cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/btf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/filter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/rbtree.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <uapi/linux/btf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #ifdef CONFIG_CGROUP_BPF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) DEFINE_PER_CPU(struct bpf_cgroup_storage_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "../cgroup/cgroup-internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #define LOCAL_STORAGE_CREATE_FLAG_MASK \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) struct bpf_cgroup_storage_map {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) struct bpf_map map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct rb_root root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) return container_of(map, struct bpf_cgroup_storage_map, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static bool attach_type_isolated(const struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) return map->key_size == sizeof(struct bpf_cgroup_storage_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static int bpf_cgroup_storage_key_cmp(const struct bpf_cgroup_storage_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) const void *_key1, const void *_key2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) if (attach_type_isolated(&map->map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) const struct bpf_cgroup_storage_key *key1 = _key1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) const struct bpf_cgroup_storage_key *key2 = _key2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) if (key1->cgroup_inode_id < key2->cgroup_inode_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) else if (key1->cgroup_inode_id > key2->cgroup_inode_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) else if (key1->attach_type < key2->attach_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) else if (key1->attach_type > key2->attach_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) const __u64 *cgroup_inode_id1 = _key1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) const __u64 *cgroup_inode_id2 = _key2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) if (*cgroup_inode_id1 < *cgroup_inode_id2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) else if (*cgroup_inode_id1 > *cgroup_inode_id2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct bpf_cgroup_storage *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) cgroup_storage_lookup(struct bpf_cgroup_storage_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) void *key, bool locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct rb_root *root = &map->root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if (!locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) spin_lock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) node = root->rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) while (node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct bpf_cgroup_storage *storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) storage = container_of(node, struct bpf_cgroup_storage, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) case -1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) node = node->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) case 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) node = node->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if (!locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) spin_unlock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) return storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) if (!locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) spin_unlock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct bpf_cgroup_storage *storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) struct rb_root *root = &map->root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) struct rb_node **new = &(root->rb_node), *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) while (*new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) struct bpf_cgroup_storage *this;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) this = container_of(*new, struct bpf_cgroup_storage, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) parent = *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) case -1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) new = &((*new)->rb_left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) case 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) new = &((*new)->rb_right);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) rb_link_node(&storage->node, parent, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) rb_insert_color(&storage->node, root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static void *cgroup_storage_lookup_elem(struct bpf_map *_map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) struct bpf_cgroup_storage_map *map = map_to_storage(_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct bpf_cgroup_storage *storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) storage = cgroup_storage_lookup(map, key, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (!storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) return &READ_ONCE(storage->buf)->data[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static int cgroup_storage_update_elem(struct bpf_map *map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) void *value, u64 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct bpf_cgroup_storage *storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) struct bpf_storage_buffer *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (unlikely(flags & ~(BPF_F_LOCK | BPF_EXIST)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (unlikely((flags & BPF_F_LOCK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) !map_value_has_spin_lock(map)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) key, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (!storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (flags & BPF_F_LOCK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) copy_map_value_locked(map, storage->buf->data, value, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) new = kmalloc_node(sizeof(struct bpf_storage_buffer) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) map->value_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) __GFP_ZERO | GFP_ATOMIC | __GFP_NOWARN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) map->numa_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) memcpy(&new->data[0], value, map->value_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) check_and_init_map_lock(map, new->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) new = xchg(&storage->buf, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) kfree_rcu(new, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) void *value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct bpf_cgroup_storage_map *map = map_to_storage(_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) struct bpf_cgroup_storage *storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) int cpu, off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) u32 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) storage = cgroup_storage_lookup(map, key, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) if (!storage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* per_cpu areas are zero-filled and bpf programs can only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * access 'value_size' of them, so copying rounded areas
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * will not leak any kernel data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) size = round_up(_map->value_size, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) bpf_long_memcpy(value + off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) per_cpu_ptr(storage->percpu_buf, cpu), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) off += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) void *value, u64 map_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) struct bpf_cgroup_storage_map *map = map_to_storage(_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct bpf_cgroup_storage *storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) int cpu, off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) u32 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (map_flags != BPF_ANY && map_flags != BPF_EXIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) storage = cgroup_storage_lookup(map, key, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (!storage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) /* the user space will provide round_up(value_size, 8) bytes that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * will be copied into per-cpu area. bpf programs can only access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * value_size of it. During lookup the same extra bytes will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * returned or zeros which were zero-filled by percpu_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * so no kernel data leaks possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) size = round_up(_map->value_size, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) value + off, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) off += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) static int cgroup_storage_get_next_key(struct bpf_map *_map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) void *_next_key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct bpf_cgroup_storage_map *map = map_to_storage(_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) struct bpf_cgroup_storage *storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) spin_lock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (list_empty(&map->list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) goto enoent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (key) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) storage = cgroup_storage_lookup(map, key, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (!storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) goto enoent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) storage = list_next_entry(storage, list_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (!storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) goto enoent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) storage = list_first_entry(&map->list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct bpf_cgroup_storage, list_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) spin_unlock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) if (attach_type_isolated(&map->map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct bpf_cgroup_storage_key *next = _next_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) *next = storage->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) __u64 *next = _next_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) *next = storage->key.cgroup_inode_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) enoent:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) spin_unlock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) int numa_node = bpf_map_attr_numa_node(attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) struct bpf_cgroup_storage_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) struct bpf_map_memory mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) if (attr->key_size != sizeof(struct bpf_cgroup_storage_key) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) attr->key_size != sizeof(__u64))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (attr->value_size == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (attr->value_size > PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) return ERR_PTR(-E2BIG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (attr->map_flags & ~LOCAL_STORAGE_CREATE_FLAG_MASK ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) !bpf_map_flags_access_ok(attr->map_flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (attr->max_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) /* max_entries is not used and enforced to be 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) ret = bpf_map_charge_init(&mem, sizeof(struct bpf_cgroup_storage_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) __GFP_ZERO | GFP_USER, numa_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (!map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) bpf_map_charge_finish(&mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) bpf_map_charge_move(&map->map.memory, &mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) /* copy mandatory map attributes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) bpf_map_init_from_attr(&map->map, attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) spin_lock_init(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) map->root = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) INIT_LIST_HEAD(&map->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return &map->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static void cgroup_storage_map_free(struct bpf_map *_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) struct bpf_cgroup_storage_map *map = map_to_storage(_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) struct list_head *storages = &map->list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) struct bpf_cgroup_storage *storage, *stmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) mutex_lock(&cgroup_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) list_for_each_entry_safe(storage, stmp, storages, list_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) bpf_cgroup_storage_unlink(storage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) bpf_cgroup_storage_free(storage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) mutex_unlock(&cgroup_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) WARN_ON(!RB_EMPTY_ROOT(&map->root));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) WARN_ON(!list_empty(&map->list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) kfree(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static int cgroup_storage_delete_elem(struct bpf_map *map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) static int cgroup_storage_check_btf(const struct bpf_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) const struct btf *btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) const struct btf_type *key_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) const struct btf_type *value_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) if (attach_type_isolated(map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct btf_member *m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) u32 offset, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) /* Key is expected to be of struct bpf_cgroup_storage_key type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * which is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * struct bpf_cgroup_storage_key {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * __u64 cgroup_inode_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * __u32 attach_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * Key_type must be a structure with two fields.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (BTF_INFO_KIND(key_type->info) != BTF_KIND_STRUCT ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) BTF_INFO_VLEN(key_type->info) != 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * The first field must be a 64 bit integer at 0 offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) m = (struct btf_member *)(key_type + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) size = sizeof_field(struct bpf_cgroup_storage_key, cgroup_inode_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) if (!btf_member_is_reg_int(btf, key_type, m, 0, size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * The second field must be a 32 bit integer at 64 bit offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) m++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) offset = offsetof(struct bpf_cgroup_storage_key, attach_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) size = sizeof_field(struct bpf_cgroup_storage_key, attach_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (!btf_member_is_reg_int(btf, key_type, m, offset, size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) u32 int_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * Key is expected to be u64, which stores the cgroup_inode_id
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) int_data = *(u32 *)(key_type + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (BTF_INT_BITS(int_data) != 64 || BTF_INT_OFFSET(int_data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) static void cgroup_storage_seq_show_elem(struct bpf_map *map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) struct bpf_cgroup_storage *storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) storage = cgroup_storage_lookup(map_to_storage(map), key, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (!storage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) stype = cgroup_storage_type(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) if (stype == BPF_CGROUP_STORAGE_SHARED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) seq_puts(m, ": ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) btf_type_seq_show(map->btf, map->btf_value_type_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) &READ_ONCE(storage->buf)->data[0], m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) seq_puts(m, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) seq_puts(m, ": {\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) seq_printf(m, "\tcpu%d: ", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) btf_type_seq_show(map->btf, map->btf_value_type_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) per_cpu_ptr(storage->percpu_buf, cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) seq_puts(m, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) seq_puts(m, "}\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) static int cgroup_storage_map_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) const struct bpf_map_ops cgroup_storage_map_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) .map_alloc = cgroup_storage_map_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) .map_free = cgroup_storage_map_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) .map_get_next_key = cgroup_storage_get_next_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) .map_lookup_elem = cgroup_storage_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) .map_update_elem = cgroup_storage_update_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) .map_delete_elem = cgroup_storage_delete_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) .map_check_btf = cgroup_storage_check_btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) .map_seq_show_elem = cgroup_storage_seq_show_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) .map_btf_name = "bpf_cgroup_storage_map",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) .map_btf_id = &cgroup_storage_map_btf_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux, struct bpf_map *_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) if (aux->cgroup_storage[stype] &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) aux->cgroup_storage[stype] != _map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) aux->cgroup_storage[stype] = _map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) static size_t bpf_cgroup_storage_calculate_size(struct bpf_map *map, u32 *pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (cgroup_storage_type(map) == BPF_CGROUP_STORAGE_SHARED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) size = sizeof(struct bpf_storage_buffer) + map->value_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) *pages = round_up(sizeof(struct bpf_cgroup_storage) + size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) PAGE_SIZE) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) size = map->value_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) *pages = round_up(round_up(size, 8) * num_possible_cpus(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) PAGE_SIZE) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) enum bpf_cgroup_storage_type stype)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) struct bpf_cgroup_storage *storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) struct bpf_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) gfp_t flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) u32 pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) map = prog->aux->cgroup_storage[stype];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) if (!map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) size = bpf_cgroup_storage_calculate_size(map, &pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (bpf_map_charge_memlock(map, pages))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) return ERR_PTR(-EPERM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) storage = kmalloc_node(sizeof(struct bpf_cgroup_storage),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) __GFP_ZERO | GFP_USER, map->numa_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if (!storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) goto enomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) flags = __GFP_ZERO | GFP_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (stype == BPF_CGROUP_STORAGE_SHARED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) storage->buf = kmalloc_node(size, flags, map->numa_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (!storage->buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) goto enomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) check_and_init_map_lock(map, storage->buf->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) storage->percpu_buf = __alloc_percpu_gfp(size, 8, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (!storage->percpu_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) goto enomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) storage->map = (struct bpf_cgroup_storage_map *)map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) return storage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) enomem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) bpf_map_uncharge_memlock(map, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) kfree(storage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) static void free_shared_cgroup_storage_rcu(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct bpf_cgroup_storage *storage =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) container_of(rcu, struct bpf_cgroup_storage, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) kfree(storage->buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) kfree(storage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) static void free_percpu_cgroup_storage_rcu(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) struct bpf_cgroup_storage *storage =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) container_of(rcu, struct bpf_cgroup_storage, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) free_percpu(storage->percpu_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) kfree(storage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) enum bpf_cgroup_storage_type stype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct bpf_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) u32 pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (!storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) map = &storage->map->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) bpf_cgroup_storage_calculate_size(map, &pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) bpf_map_uncharge_memlock(map, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) stype = cgroup_storage_type(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (stype == BPF_CGROUP_STORAGE_SHARED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) struct cgroup *cgroup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) enum bpf_attach_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) struct bpf_cgroup_storage_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) if (!storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) storage->key.attach_type = type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) storage->key.cgroup_inode_id = cgroup_id(cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) map = storage->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) spin_lock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) WARN_ON(cgroup_storage_insert(map, storage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) list_add(&storage->list_map, &map->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) list_add(&storage->list_cg, &cgroup->bpf.storages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) spin_unlock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) struct bpf_cgroup_storage_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct rb_root *root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (!storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) map = storage->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) spin_lock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) root = &map->root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) rb_erase(&storage->node, root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) list_del(&storage->list_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) list_del(&storage->list_cg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) spin_unlock_bh(&map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags