^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (c) 2016,2017 Facebook
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/btf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/filter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <uapi/linux/btf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/rcupdate_trace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "map_in_map.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define ARRAY_CREATE_FLAG_MASK \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) (BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) BPF_F_PRESERVE_ELEMS | BPF_F_INNER_MAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) static void bpf_array_free_percpu(struct bpf_array *array)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) for (i = 0; i < array->map.max_entries; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) free_percpu(array->pptrs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) static int bpf_array_alloc_percpu(struct bpf_array *array)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) void __percpu *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) for (i = 0; i < array->map.max_entries; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) ptr = __alloc_percpu_gfp(array->elem_size, 8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) GFP_USER | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (!ptr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) bpf_array_free_percpu(array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) array->pptrs[i] = ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) /* Called from syscall */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) int array_map_alloc_check(union bpf_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) int numa_node = bpf_map_attr_numa_node(attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /* check sanity of attributes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) if (attr->max_entries == 0 || attr->key_size != 4 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) attr->value_size == 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) attr->map_flags & ~ARRAY_CREATE_FLAG_MASK ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) !bpf_map_flags_access_ok(attr->map_flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) (percpu && numa_node != NUMA_NO_NODE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) attr->map_flags & (BPF_F_MMAPABLE | BPF_F_INNER_MAP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) attr->map_flags & BPF_F_PRESERVE_ELEMS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (attr->value_size > KMALLOC_MAX_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* if value_size is bigger, the user space won't be able to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * access the elements.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static struct bpf_map *array_map_alloc(union bpf_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) int ret, numa_node = bpf_map_attr_numa_node(attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) u32 elem_size, index_mask, max_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) bool bypass_spec_v1 = bpf_bypass_spec_v1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) u64 cost, array_size, mask64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct bpf_map_memory mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct bpf_array *array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) elem_size = round_up(attr->value_size, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) max_entries = attr->max_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) /* On 32 bit archs roundup_pow_of_two() with max_entries that has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * upper most bit set in u32 space is undefined behavior due to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * resulting 1U << 32, so do it manually here in u64 space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) mask64 = fls_long(max_entries - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) mask64 = 1ULL << mask64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) mask64 -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) index_mask = mask64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (!bypass_spec_v1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /* round up array size to nearest power of 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * since cpu will speculate within index_mask limits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) max_entries = index_mask + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) /* Check for overflows. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (max_entries < attr->max_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return ERR_PTR(-E2BIG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) array_size = sizeof(*array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (percpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) array_size += (u64) max_entries * sizeof(void *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) /* rely on vmalloc() to return page-aligned memory and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * ensure array->value is exactly page-aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (attr->map_flags & BPF_F_MMAPABLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) array_size = PAGE_ALIGN(array_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) array_size += PAGE_ALIGN((u64) max_entries * elem_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) array_size += (u64) max_entries * elem_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /* make sure there is no u32 overflow later in round_up() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) cost = array_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) cost += (u64)attr->max_entries * elem_size * num_possible_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) ret = bpf_map_charge_init(&mem, cost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /* allocate all map elements and zero-initialize them */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (attr->map_flags & BPF_F_MMAPABLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) void *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) /* kmalloc'ed memory can't be mmap'ed, use explicit vmalloc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) data = bpf_map_area_mmapable_alloc(array_size, numa_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) if (!data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) bpf_map_charge_finish(&mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) array = data + PAGE_ALIGN(sizeof(struct bpf_array))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) - offsetof(struct bpf_array, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) array = bpf_map_area_alloc(array_size, numa_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) if (!array) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) bpf_map_charge_finish(&mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) array->index_mask = index_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) array->map.bypass_spec_v1 = bypass_spec_v1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) /* copy mandatory map attributes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) bpf_map_init_from_attr(&array->map, attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) bpf_map_charge_move(&array->map.memory, &mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) array->elem_size = elem_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (percpu && bpf_array_alloc_percpu(array)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) bpf_map_charge_finish(&array->map.memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) bpf_map_area_free(array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return &array->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /* Called from syscall or from eBPF program */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static void *array_map_lookup_elem(struct bpf_map *map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) u32 index = *(u32 *)key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (unlikely(index >= array->map.max_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return array->value + array->elem_size * (index & array->index_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) u32 off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (map->max_entries != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (off >= map->value_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) *imm = (unsigned long)array->value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) u32 *off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) u64 base = (unsigned long)array->value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) u64 range = array->elem_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (map->max_entries != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if (imm < base || imm >= base + range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) *off = imm - base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) struct bpf_insn *insn = insn_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) u32 elem_size = round_up(map->value_size, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) const int ret = BPF_REG_0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) const int map_ptr = BPF_REG_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) const int index = BPF_REG_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (map->map_flags & BPF_F_INNER_MAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) if (!map->bypass_spec_v1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (is_power_of_2(elem_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) *insn++ = BPF_MOV64_IMM(ret, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return insn - insn_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /* Called from eBPF program */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) u32 index = *(u32 *)key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (unlikely(index >= array->map.max_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return this_cpu_ptr(array->pptrs[index & array->index_mask]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) u32 index = *(u32 *)key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) void __percpu *pptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) int cpu, off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) u32 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) if (unlikely(index >= array->map.max_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /* per_cpu areas are zero-filled and bpf programs can only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * access 'value_size' of them, so copying rounded areas
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * will not leak any kernel data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) size = round_up(map->value_size, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) pptr = array->pptrs[index & array->index_mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) bpf_long_memcpy(value + off, per_cpu_ptr(pptr, cpu), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) off += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) /* Called from syscall */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) u32 index = key ? *(u32 *)key : U32_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) u32 *next = (u32 *)next_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) if (index >= array->map.max_entries) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) *next = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) if (index == array->map.max_entries - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) *next = index + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /* Called from syscall or from eBPF program */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) static int array_map_update_elem(struct bpf_map *map, void *key, void *value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) u64 map_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) u32 index = *(u32 *)key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) char *val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) /* unknown flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (unlikely(index >= array->map.max_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) /* all elements were pre-allocated, cannot insert a new one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (unlikely(map_flags & BPF_NOEXIST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /* all elements already exist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (unlikely((map_flags & BPF_F_LOCK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) !map_value_has_spin_lock(map)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) value, map->value_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) val = array->value +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) array->elem_size * (index & array->index_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (map_flags & BPF_F_LOCK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) copy_map_value_locked(map, val, value, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) copy_map_value(map, val, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) u64 map_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) u32 index = *(u32 *)key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) void __percpu *pptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) int cpu, off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) u32 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) if (unlikely(map_flags > BPF_EXIST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /* unknown flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (unlikely(index >= array->map.max_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /* all elements were pre-allocated, cannot insert a new one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (unlikely(map_flags == BPF_NOEXIST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) /* all elements already exist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) /* the user space will provide round_up(value_size, 8) bytes that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * will be copied into per-cpu area. bpf programs can only access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * value_size of it. During lookup the same extra bytes will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * returned or zeros which were zero-filled by percpu_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * so no kernel data leaks possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) size = round_up(map->value_size, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) pptr = array->pptrs[index & array->index_mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) off += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) /* Called from syscall or from eBPF program */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) static int array_map_delete_elem(struct bpf_map *map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) static void *array_map_vmalloc_addr(struct bpf_array *array)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) return (void *)round_down((unsigned long)array, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) static void array_map_free(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) bpf_array_free_percpu(array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (array->map.map_flags & BPF_F_MMAPABLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) bpf_map_area_free(array_map_vmalloc_addr(array));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) bpf_map_area_free(array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static void array_map_seq_show_elem(struct bpf_map *map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) void *value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) value = array_map_lookup_elem(map, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (!value) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (map->btf_key_type_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) seq_printf(m, "%u: ", *(u32 *)key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) seq_puts(m, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) u32 index = *(u32 *)key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) void __percpu *pptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) seq_printf(m, "%u: {\n", *(u32 *)key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) pptr = array->pptrs[index & array->index_mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) seq_printf(m, "\tcpu%d: ", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) btf_type_seq_show(map->btf, map->btf_value_type_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) per_cpu_ptr(pptr, cpu), m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) seq_puts(m, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) seq_puts(m, "}\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) static int array_map_check_btf(const struct bpf_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) const struct btf *btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) const struct btf_type *key_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) const struct btf_type *value_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) u32 int_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /* One exception for keyless BTF: .bss/.data/.rodata map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (btf_type_is_void(key_type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (map->map_type != BPF_MAP_TYPE_ARRAY ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) map->max_entries != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (BTF_INFO_KIND(value_type->info) != BTF_KIND_DATASEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) int_data = *(u32 *)(key_type + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) /* bpf array can only take a u32 key. This check makes sure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * that the btf matches the attr used during map_create.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (!(map->map_flags & BPF_F_MMAPABLE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) PAGE_ALIGN((u64)array->map.max_entries * array->elem_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) return remap_vmalloc_range(vma, array_map_vmalloc_addr(array),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) vma->vm_pgoff + pgoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) static bool array_map_meta_equal(const struct bpf_map *meta0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) const struct bpf_map *meta1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (!bpf_map_meta_equal(meta0, meta1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) return meta0->map_flags & BPF_F_INNER_MAP ? true :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) meta0->max_entries == meta1->max_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) struct bpf_iter_seq_array_map_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) struct bpf_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) void *percpu_value_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) u32 index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) static void *bpf_array_map_seq_start(struct seq_file *seq, loff_t *pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) struct bpf_iter_seq_array_map_info *info = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) struct bpf_map *map = info->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) struct bpf_array *array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) u32 index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (info->index >= map->max_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (*pos == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) ++*pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) index = info->index & array->index_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) if (info->percpu_value_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) return array->pptrs[index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) return array->value + array->elem_size * index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) static void *bpf_array_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct bpf_iter_seq_array_map_info *info = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) struct bpf_map *map = info->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) struct bpf_array *array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) u32 index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) ++*pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) ++info->index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (info->index >= map->max_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) index = info->index & array->index_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (info->percpu_value_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return array->pptrs[index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) return array->value + array->elem_size * index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) static int __bpf_array_map_seq_show(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) struct bpf_iter_seq_array_map_info *info = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) struct bpf_iter__bpf_map_elem ctx = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) struct bpf_map *map = info->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) struct bpf_iter_meta meta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) struct bpf_prog *prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) int off = 0, cpu = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) void __percpu **pptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) u32 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) meta.seq = seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) prog = bpf_iter_get_info(&meta, v == NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (!prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) ctx.meta = &meta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) ctx.map = info->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) if (v) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) ctx.key = &info->index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (!info->percpu_value_buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) ctx.value = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) pptr = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) size = round_up(map->value_size, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) bpf_long_memcpy(info->percpu_value_buf + off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) per_cpu_ptr(pptr, cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) off += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) ctx.value = info->percpu_value_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return bpf_iter_run_prog(prog, &ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) static int bpf_array_map_seq_show(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) return __bpf_array_map_seq_show(seq, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) static void bpf_array_map_seq_stop(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (!v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) (void)__bpf_array_map_seq_show(seq, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) static int bpf_iter_init_array_map(void *priv_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) struct bpf_iter_aux_info *aux)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) struct bpf_iter_seq_array_map_info *seq_info = priv_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct bpf_map *map = aux->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) void *value_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) u32 buf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) buf_size = round_up(map->value_size, 8) * num_possible_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) value_buf = kmalloc(buf_size, GFP_USER | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) if (!value_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) seq_info->percpu_value_buf = value_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) seq_info->map = map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) static void bpf_iter_fini_array_map(void *priv_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) struct bpf_iter_seq_array_map_info *seq_info = priv_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) kfree(seq_info->percpu_value_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) static const struct seq_operations bpf_array_map_seq_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) .start = bpf_array_map_seq_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) .next = bpf_array_map_seq_next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) .stop = bpf_array_map_seq_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) .show = bpf_array_map_seq_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) static const struct bpf_iter_seq_info iter_seq_info = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) .seq_ops = &bpf_array_map_seq_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) .init_seq_private = bpf_iter_init_array_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) .fini_seq_private = bpf_iter_fini_array_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) .seq_priv_size = sizeof(struct bpf_iter_seq_array_map_info),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) static int array_map_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) const struct bpf_map_ops array_map_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) .map_meta_equal = array_map_meta_equal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) .map_alloc_check = array_map_alloc_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) .map_alloc = array_map_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) .map_free = array_map_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) .map_get_next_key = array_map_get_next_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) .map_lookup_elem = array_map_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) .map_update_elem = array_map_update_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) .map_delete_elem = array_map_delete_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) .map_gen_lookup = array_map_gen_lookup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) .map_direct_value_addr = array_map_direct_value_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) .map_direct_value_meta = array_map_direct_value_meta,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) .map_mmap = array_map_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) .map_seq_show_elem = array_map_seq_show_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) .map_check_btf = array_map_check_btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) .map_lookup_batch = generic_map_lookup_batch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) .map_update_batch = generic_map_update_batch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) .map_btf_name = "bpf_array",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) .map_btf_id = &array_map_btf_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) .iter_seq_info = &iter_seq_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) static int percpu_array_map_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) const struct bpf_map_ops percpu_array_map_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) .map_meta_equal = bpf_map_meta_equal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) .map_alloc_check = array_map_alloc_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) .map_alloc = array_map_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) .map_free = array_map_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) .map_get_next_key = array_map_get_next_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) .map_lookup_elem = percpu_array_map_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) .map_update_elem = array_map_update_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) .map_delete_elem = array_map_delete_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) .map_seq_show_elem = percpu_array_map_seq_show_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) .map_check_btf = array_map_check_btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) .map_btf_name = "bpf_array",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) .map_btf_id = &percpu_array_map_btf_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) .iter_seq_info = &iter_seq_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) static int fd_array_map_alloc_check(union bpf_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) /* only file descriptors can be stored in this type of map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (attr->value_size != sizeof(u32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) /* Program read-only/write-only not supported for special maps yet. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) return array_map_alloc_check(attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) static void fd_array_map_free(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) /* make sure it's empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) for (i = 0; i < array->map.max_entries; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) BUG_ON(array->ptrs[i] != NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) bpf_map_area_free(array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return ERR_PTR(-EOPNOTSUPP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) /* only called from syscall */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) void **elem, *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) if (!map->ops->map_fd_sys_lookup_elem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) elem = array_map_lookup_elem(map, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) if (elem && (ptr = READ_ONCE(*elem)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) *value = map->ops->map_fd_sys_lookup_elem(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) ret = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) /* only called from syscall */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) void *key, void *value, u64 map_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) void *new_ptr, *old_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) u32 index = *(u32 *)key, ufd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) if (map_flags != BPF_ANY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) if (index >= array->map.max_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) ufd = *(u32 *)value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) if (IS_ERR(new_ptr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return PTR_ERR(new_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) if (map->ops->map_poke_run) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) mutex_lock(&array->aux->poke_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) old_ptr = xchg(array->ptrs + index, new_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) map->ops->map_poke_run(map, index, old_ptr, new_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) mutex_unlock(&array->aux->poke_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) old_ptr = xchg(array->ptrs + index, new_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) if (old_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) map->ops->map_fd_put_ptr(old_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) static int fd_array_map_delete_elem(struct bpf_map *map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) void *old_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) u32 index = *(u32 *)key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (index >= array->map.max_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) if (map->ops->map_poke_run) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) mutex_lock(&array->aux->poke_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) old_ptr = xchg(array->ptrs + index, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) map->ops->map_poke_run(map, index, old_ptr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) mutex_unlock(&array->aux->poke_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) old_ptr = xchg(array->ptrs + index, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (old_ptr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) map->ops->map_fd_put_ptr(old_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) static void *prog_fd_array_get_ptr(struct bpf_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) struct file *map_file, int fd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) struct bpf_prog *prog = bpf_prog_get(fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (IS_ERR(prog))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) return prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) if (!bpf_prog_array_compatible(array, prog)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) bpf_prog_put(prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) return prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) static void prog_fd_array_put_ptr(void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) bpf_prog_put(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) static u32 prog_fd_array_sys_lookup_elem(void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) return ((struct bpf_prog *)ptr)->aux->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) /* decrement refcnt of all bpf_progs that are stored in this map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) static void bpf_fd_array_map_clear(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) for (i = 0; i < array->map.max_entries; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) fd_array_map_delete_elem(map, &i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) void **elem, *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) u32 prog_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) elem = array_map_lookup_elem(map, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) if (elem) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) ptr = READ_ONCE(*elem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) if (ptr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) seq_printf(m, "%u: ", *(u32 *)key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) prog_id = prog_fd_array_sys_lookup_elem(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) btf_type_seq_show(map->btf, map->btf_value_type_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) &prog_id, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) seq_puts(m, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) struct prog_poke_elem {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) struct bpf_prog_aux *aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) static int prog_array_map_poke_track(struct bpf_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) struct bpf_prog_aux *prog_aux)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) struct prog_poke_elem *elem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) struct bpf_array_aux *aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) aux = container_of(map, struct bpf_array, map)->aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) mutex_lock(&aux->poke_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) list_for_each_entry(elem, &aux->poke_progs, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (elem->aux == prog_aux)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) elem = kmalloc(sizeof(*elem), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) if (!elem) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) INIT_LIST_HEAD(&elem->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) /* We must track the program's aux info at this point in time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * since the program pointer itself may not be stable yet, see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * also comment in prog_array_map_poke_run().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) elem->aux = prog_aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) list_add_tail(&elem->list, &aux->poke_progs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) mutex_unlock(&aux->poke_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) static void prog_array_map_poke_untrack(struct bpf_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) struct bpf_prog_aux *prog_aux)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) struct prog_poke_elem *elem, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) struct bpf_array_aux *aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) aux = container_of(map, struct bpf_array, map)->aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) mutex_lock(&aux->poke_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) if (elem->aux == prog_aux) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) list_del_init(&elem->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) kfree(elem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) mutex_unlock(&aux->poke_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) static void prog_array_map_poke_run(struct bpf_map *map, u32 key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) struct bpf_prog *old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) struct bpf_prog *new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) u8 *old_addr, *new_addr, *old_bypass_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) struct prog_poke_elem *elem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) struct bpf_array_aux *aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) aux = container_of(map, struct bpf_array, map)->aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) WARN_ON_ONCE(!mutex_is_locked(&aux->poke_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) list_for_each_entry(elem, &aux->poke_progs, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) struct bpf_jit_poke_descriptor *poke;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) for (i = 0; i < elem->aux->size_poke_tab; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) poke = &elem->aux->poke_tab[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) /* Few things to be aware of:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) * 1) We can only ever access aux in this context, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * not aux->prog since it might not be stable yet and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * there could be danger of use after free otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) * 2) Initially when we start tracking aux, the program
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) * is not JITed yet and also does not have a kallsyms
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) * entry. We skip these as poke->tailcall_target_stable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) * is not active yet. The JIT will do the final fixup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) * before setting it stable. The various
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) * poke->tailcall_target_stable are successively
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) * activated, so tail call updates can arrive from here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) * while JIT is still finishing its final fixup for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) * non-activated poke entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) * 3) On program teardown, the program's kallsym entry gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * removed out of RCU callback, but we can only untrack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * from sleepable context, therefore bpf_arch_text_poke()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) * might not see that this is in BPF text section and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * bails out with -EINVAL. As these are unreachable since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) * RCU grace period already passed, we simply skip them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) * 4) Also programs reaching refcount of zero while patching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) * is in progress is okay since we're protected under
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) * poke_mutex and untrack the programs before the JIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) * buffer is freed. When we're still in the middle of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) * patching and suddenly kallsyms entry of the program
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * gets evicted, we just skip the rest which is fine due
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) * to point 3).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * 5) Any other error happening below from bpf_arch_text_poke()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * is a unexpected bug.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) if (!READ_ONCE(poke->tailcall_target_stable))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) if (poke->reason != BPF_POKE_REASON_TAIL_CALL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) if (poke->tail_call.map != map ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) poke->tail_call.key != key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) old_bypass_addr = old ? NULL : poke->bypass_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) old_addr = old ? (u8 *)old->bpf_func + poke->adj_off : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) new_addr = new ? (u8 *)new->bpf_func + poke->adj_off : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) if (new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) ret = bpf_arch_text_poke(poke->tailcall_target,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) BPF_MOD_JUMP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) old_addr, new_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) BUG_ON(ret < 0 && ret != -EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (!old) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) ret = bpf_arch_text_poke(poke->tailcall_bypass,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) BPF_MOD_JUMP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) poke->bypass_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) BUG_ON(ret < 0 && ret != -EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) ret = bpf_arch_text_poke(poke->tailcall_bypass,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) BPF_MOD_JUMP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) old_bypass_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) poke->bypass_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) BUG_ON(ret < 0 && ret != -EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) /* let other CPUs finish the execution of program
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) * so that it will not possible to expose them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) * to invalid nop, stack unwind, nop state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) ret = bpf_arch_text_poke(poke->tailcall_target,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) BPF_MOD_JUMP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) old_addr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) BUG_ON(ret < 0 && ret != -EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) static void prog_array_map_clear_deferred(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) struct bpf_map *map = container_of(work, struct bpf_array_aux,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) work)->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) bpf_fd_array_map_clear(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) bpf_map_put(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) static void prog_array_map_clear(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) struct bpf_array_aux *aux = container_of(map, struct bpf_array,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) map)->aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) bpf_map_inc(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) schedule_work(&aux->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) static struct bpf_map *prog_array_map_alloc(union bpf_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) struct bpf_array_aux *aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) struct bpf_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) aux = kzalloc(sizeof(*aux), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) if (!aux)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) INIT_WORK(&aux->work, prog_array_map_clear_deferred);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) INIT_LIST_HEAD(&aux->poke_progs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) mutex_init(&aux->poke_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) spin_lock_init(&aux->owner.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) map = array_map_alloc(attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if (IS_ERR(map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) kfree(aux);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) container_of(map, struct bpf_array, map)->aux = aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) aux->map = map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) return map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) static void prog_array_map_free(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) struct prog_poke_elem *elem, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) struct bpf_array_aux *aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) aux = container_of(map, struct bpf_array, map)->aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) list_del_init(&elem->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) kfree(elem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) kfree(aux);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) fd_array_map_free(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) /* prog_array->aux->{type,jited} is a runtime binding.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) * Doing static check alone in the verifier is not enough.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) * Thus, prog_array_map cannot be used as an inner_map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) * and map_meta_equal is not implemented.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) static int prog_array_map_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) const struct bpf_map_ops prog_array_map_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) .map_alloc_check = fd_array_map_alloc_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) .map_alloc = prog_array_map_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) .map_free = prog_array_map_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) .map_poke_track = prog_array_map_poke_track,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) .map_poke_untrack = prog_array_map_poke_untrack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) .map_poke_run = prog_array_map_poke_run,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) .map_get_next_key = array_map_get_next_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) .map_lookup_elem = fd_array_map_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) .map_delete_elem = fd_array_map_delete_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) .map_fd_get_ptr = prog_fd_array_get_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) .map_fd_put_ptr = prog_fd_array_put_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) .map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) .map_release_uref = prog_array_map_clear,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) .map_seq_show_elem = prog_array_map_seq_show_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) .map_btf_name = "bpf_array",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) .map_btf_id = &prog_array_map_btf_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) struct file *map_file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) struct bpf_event_entry *ee;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) ee = kzalloc(sizeof(*ee), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) if (ee) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) ee->event = perf_file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) ee->perf_file = perf_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) ee->map_file = map_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) return ee;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) static void __bpf_event_entry_free(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) struct bpf_event_entry *ee;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) ee = container_of(rcu, struct bpf_event_entry, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) fput(ee->perf_file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) kfree(ee);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) call_rcu(&ee->rcu, __bpf_event_entry_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) static void *perf_event_fd_array_get_ptr(struct bpf_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) struct file *map_file, int fd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) struct bpf_event_entry *ee;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) struct perf_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) struct file *perf_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) u64 value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) perf_file = perf_event_get(fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) if (IS_ERR(perf_file))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) return perf_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) ee = ERR_PTR(-EOPNOTSUPP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) event = perf_file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) goto err_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) ee = bpf_event_entry_gen(perf_file, map_file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) if (ee)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) return ee;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) ee = ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) err_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) fput(perf_file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) return ee;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) static void perf_event_fd_array_put_ptr(void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) bpf_event_entry_free_rcu(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) static void perf_event_fd_array_release(struct bpf_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) struct file *map_file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) struct bpf_event_entry *ee;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) if (map->map_flags & BPF_F_PRESERVE_ELEMS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) for (i = 0; i < array->map.max_entries; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) ee = READ_ONCE(array->ptrs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) if (ee && ee->map_file == map_file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) fd_array_map_delete_elem(map, &i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) static void perf_event_fd_array_map_free(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) if (map->map_flags & BPF_F_PRESERVE_ELEMS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) bpf_fd_array_map_clear(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) fd_array_map_free(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) static int perf_event_array_map_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) const struct bpf_map_ops perf_event_array_map_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) .map_meta_equal = bpf_map_meta_equal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) .map_alloc_check = fd_array_map_alloc_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) .map_alloc = array_map_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) .map_free = perf_event_fd_array_map_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) .map_get_next_key = array_map_get_next_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) .map_lookup_elem = fd_array_map_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) .map_delete_elem = fd_array_map_delete_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) .map_fd_get_ptr = perf_event_fd_array_get_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) .map_fd_put_ptr = perf_event_fd_array_put_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) .map_release = perf_event_fd_array_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) .map_check_btf = map_check_no_btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) .map_btf_name = "bpf_array",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) .map_btf_id = &perf_event_array_map_btf_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) #ifdef CONFIG_CGROUPS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) static void *cgroup_fd_array_get_ptr(struct bpf_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) struct file *map_file /* not used */,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) int fd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) return cgroup_get_from_fd(fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) static void cgroup_fd_array_put_ptr(void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) /* cgroup_put free cgrp after a rcu grace period */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) cgroup_put(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) static void cgroup_fd_array_free(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) bpf_fd_array_map_clear(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) fd_array_map_free(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) static int cgroup_array_map_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) const struct bpf_map_ops cgroup_array_map_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) .map_meta_equal = bpf_map_meta_equal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) .map_alloc_check = fd_array_map_alloc_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) .map_alloc = array_map_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) .map_free = cgroup_fd_array_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) .map_get_next_key = array_map_get_next_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) .map_lookup_elem = fd_array_map_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) .map_delete_elem = fd_array_map_delete_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) .map_fd_get_ptr = cgroup_fd_array_get_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) .map_fd_put_ptr = cgroup_fd_array_put_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) .map_check_btf = map_check_no_btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) .map_btf_name = "bpf_array",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) .map_btf_id = &cgroup_array_map_btf_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) static struct bpf_map *array_of_map_alloc(union bpf_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) struct bpf_map *map, *inner_map_meta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) if (IS_ERR(inner_map_meta))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) return inner_map_meta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) map = array_map_alloc(attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) if (IS_ERR(map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) bpf_map_meta_free(inner_map_meta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) return map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) map->inner_map_meta = inner_map_meta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) return map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) static void array_of_map_free(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) /* map->inner_map_meta is only accessed by syscall which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) * is protected by fdget/fdput.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) bpf_map_meta_free(map->inner_map_meta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) bpf_fd_array_map_clear(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) fd_array_map_free(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) struct bpf_map **inner_map = array_map_lookup_elem(map, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) if (!inner_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) return READ_ONCE(*inner_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) static int array_of_map_gen_lookup(struct bpf_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) struct bpf_insn *insn_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) struct bpf_array *array = container_of(map, struct bpf_array, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) u32 elem_size = round_up(map->value_size, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) struct bpf_insn *insn = insn_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) const int ret = BPF_REG_0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) const int map_ptr = BPF_REG_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) const int index = BPF_REG_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) if (!map->bypass_spec_v1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) if (is_power_of_2(elem_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) *insn++ = BPF_MOV64_IMM(ret, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) return insn - insn_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) static int array_of_maps_map_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) const struct bpf_map_ops array_of_maps_map_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) .map_alloc_check = fd_array_map_alloc_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) .map_alloc = array_of_map_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) .map_free = array_of_map_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) .map_get_next_key = array_map_get_next_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) .map_lookup_elem = array_of_map_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) .map_delete_elem = fd_array_map_delete_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) .map_fd_get_ptr = bpf_map_fd_get_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) .map_fd_put_ptr = bpf_map_fd_put_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) .map_gen_lookup = array_of_map_gen_lookup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) .map_check_btf = map_check_no_btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) .map_btf_name = "bpf_array",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) .map_btf_id = &array_of_maps_map_btf_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags