^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Copyright (c) 2019 Facebook */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/bpf_verifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/btf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/filter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/numa.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/refcount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <trace/hooks/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) enum bpf_struct_ops_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) BPF_STRUCT_OPS_STATE_INIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) BPF_STRUCT_OPS_STATE_INUSE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) BPF_STRUCT_OPS_STATE_TOBEFREE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define BPF_STRUCT_OPS_COMMON_VALUE \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) refcount_t refcnt; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) enum bpf_struct_ops_state state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) struct bpf_struct_ops_value {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) BPF_STRUCT_OPS_COMMON_VALUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) char data[] ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) struct bpf_struct_ops_map {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct bpf_map map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) const struct bpf_struct_ops *st_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /* protect map_update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) struct mutex lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /* progs has all the bpf_prog that is populated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * to the func ptr of the kernel's struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * (in kvalue.data).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct bpf_prog **progs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /* image is a page that has all the trampolines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * that stores the func args before calling the bpf_prog.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * A PAGE_SIZE "image" is enough to store all trampoline for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * "progs[]".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) void *image;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /* uvalue->data stores the kernel struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * (e.g. tcp_congestion_ops) that is more useful
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * to userspace than the kvalue. For example,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * the bpf_prog's id is stored instead of the kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * address of a func ptr.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct bpf_struct_ops_value *uvalue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /* kvalue.data stores the actual kernel's struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * (e.g. tcp_congestion_ops) that will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * registered to the kernel subsystem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct bpf_struct_ops_value kvalue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define VALUE_PREFIX "bpf_struct_ops_"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* bpf_struct_ops_##_name (e.g. bpf_struct_ops_tcp_congestion_ops) is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * the map's value exposed to the userspace and its btf-type-id is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * stored at the map->btf_vmlinux_value_type_id.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define BPF_STRUCT_OPS_TYPE(_name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) extern struct bpf_struct_ops bpf_##_name; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct bpf_struct_ops_##_name { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) BPF_STRUCT_OPS_COMMON_VALUE; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct _name data ____cacheline_aligned_in_smp; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #include "bpf_struct_ops_types.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #undef BPF_STRUCT_OPS_TYPE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #define BPF_STRUCT_OPS_TYPE(_name) BPF_STRUCT_OPS_TYPE_##_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #include "bpf_struct_ops_types.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #undef BPF_STRUCT_OPS_TYPE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) __NR_BPF_STRUCT_OPS_TYPE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static struct bpf_struct_ops * const bpf_struct_ops[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define BPF_STRUCT_OPS_TYPE(_name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) [BPF_STRUCT_OPS_TYPE_##_name] = &bpf_##_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #include "bpf_struct_ops_types.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #undef BPF_STRUCT_OPS_TYPE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static const struct btf_type *module_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) s32 type_id, value_id, module_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) const struct btf_member *member;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct bpf_struct_ops *st_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) const struct btf_type *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) char value_name[128];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) const char *mname;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) u32 i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /* Ensure BTF type is emitted for "struct bpf_struct_ops_##_name" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define BPF_STRUCT_OPS_TYPE(_name) BTF_TYPE_EMIT(struct bpf_struct_ops_##_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #include "bpf_struct_ops_types.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #undef BPF_STRUCT_OPS_TYPE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) module_id = btf_find_by_name_kind(btf, "module", BTF_KIND_STRUCT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if (module_id < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) pr_warn("Cannot find struct module in btf_vmlinux\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) module_type = btf_type_by_id(btf, module_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) st_ops = bpf_struct_ops[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (strlen(st_ops->name) + VALUE_PREFIX_LEN >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) sizeof(value_name)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) pr_warn("struct_ops name %s is too long\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) st_ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) value_id = btf_find_by_name_kind(btf, value_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) BTF_KIND_STRUCT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (value_id < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) pr_warn("Cannot find struct %s in btf_vmlinux\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) value_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) type_id = btf_find_by_name_kind(btf, st_ops->name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) BTF_KIND_STRUCT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) if (type_id < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) pr_warn("Cannot find struct %s in btf_vmlinux\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) st_ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) t = btf_type_by_id(btf, type_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) pr_warn("Cannot support #%u members in struct %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) btf_type_vlen(t), st_ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) for_each_member(j, t, member) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) const struct btf_type *func_proto;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) mname = btf_name_by_offset(btf, member->name_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (!*mname) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) pr_warn("anon member in struct %s is not supported\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) st_ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (btf_member_bitfield_size(t, member)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) pr_warn("bit field member %s in struct %s is not supported\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) mname, st_ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) func_proto = btf_type_resolve_func_ptr(btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) member->type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (func_proto &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) btf_distill_func_proto(log, btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) func_proto, mname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) &st_ops->func_models[j])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) pr_warn("Error in parsing func ptr %s in struct %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) mname, st_ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (j == btf_type_vlen(t)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) if (st_ops->init(btf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) pr_warn("Error in init bpf_struct_ops %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) st_ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) st_ops->type_id = type_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) st_ops->type = t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) st_ops->value_id = value_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) st_ops->value_type = btf_type_by_id(btf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) value_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) extern struct btf *btf_vmlinux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static const struct bpf_struct_ops *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) bpf_struct_ops_find_value(u32 value_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (!value_id || !btf_vmlinux)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (bpf_struct_ops[i]->value_id == value_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) return bpf_struct_ops[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (!type_id || !btf_vmlinux)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (bpf_struct_ops[i]->type_id == type_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return bpf_struct_ops[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) void *next_key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (key && *(u32 *)key == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) *(u32 *)next_key = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) void *value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) struct bpf_struct_ops_value *uvalue, *kvalue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) enum bpf_struct_ops_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (unlikely(*(u32 *)key != 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) kvalue = &st_map->kvalue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /* Pair with smp_store_release() during map_update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) state = smp_load_acquire(&kvalue->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (state == BPF_STRUCT_OPS_STATE_INIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) memset(value, 0, map->value_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) /* No lock is needed. state and refcnt do not need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * to be updated together under atomic context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) uvalue = (struct bpf_struct_ops_value *)value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) memcpy(uvalue, st_map->uvalue, map->value_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) uvalue->state = state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) refcount_set(&uvalue->refcnt, refcount_read(&kvalue->refcnt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) const struct btf_type *t = st_map->st_ops->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) for (i = 0; i < btf_type_vlen(t); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (st_map->progs[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) bpf_prog_put(st_map->progs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) st_map->progs[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static int check_zero_holes(const struct btf_type *t, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) const struct btf_member *member;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) u32 i, moff, msize, prev_mend = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) const struct btf_type *mtype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) for_each_member(i, t, member) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) moff = btf_member_bit_offset(t, member) / 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (moff > prev_mend &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) memchr_inv(data + prev_mend, 0, moff - prev_mend))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) mtype = btf_type_by_id(btf_vmlinux, member->type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) mtype = btf_resolve_size(btf_vmlinux, mtype, &msize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (IS_ERR(mtype))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) return PTR_ERR(mtype);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) prev_mend = moff + msize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (t->size > prev_mend &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) memchr_inv(data + prev_mend, 0, t->size - prev_mend))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) void *value, u64 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) const struct bpf_struct_ops *st_ops = st_map->st_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) struct bpf_struct_ops_value *uvalue, *kvalue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) const struct btf_member *member;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) const struct btf_type *t = st_ops->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) struct bpf_tramp_progs *tprogs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) void *udata, *kdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) int prog_fd, err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) void *image;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (*(u32 *)key != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) err = check_zero_holes(st_ops->value_type, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) uvalue = (struct bpf_struct_ops_value *)value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) err = check_zero_holes(t, uvalue->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (uvalue->state || refcount_read(&uvalue->refcnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) if (!tprogs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) mutex_lock(&st_map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (kvalue->state != BPF_STRUCT_OPS_STATE_INIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) err = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) memcpy(uvalue, value, map->value_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) udata = &uvalue->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) kdata = &kvalue->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) image = st_map->image;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) for_each_member(i, t, member) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) const struct btf_type *mtype, *ptype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) struct bpf_prog *prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) u32 moff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) u32 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) moff = btf_member_bit_offset(t, member) / 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) ptype = btf_type_resolve_ptr(btf_vmlinux, member->type, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (ptype == module_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if (*(void **)(udata + moff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) goto reset_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) *(void **)(kdata + moff) = BPF_MODULE_OWNER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) err = st_ops->init_member(t, member, kdata, udata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) goto reset_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) /* The ->init_member() has handled this member */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (err > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) /* If st_ops->init_member does not handle it,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * we will only handle func ptrs and zero-ed members
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * here. Reject everything else.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) /* All non func ptr member must be 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (!ptype || !btf_type_is_func_proto(ptype)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) u32 msize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) mtype = btf_type_by_id(btf_vmlinux, member->type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) mtype = btf_resolve_size(btf_vmlinux, mtype, &msize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (IS_ERR(mtype)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) err = PTR_ERR(mtype);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) goto reset_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (memchr_inv(udata + moff, 0, msize)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) goto reset_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) prog_fd = (int)(*(unsigned long *)(udata + moff));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) /* Similar check as the attr->attach_prog_fd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (!prog_fd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) prog = bpf_prog_get(prog_fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (IS_ERR(prog)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) err = PTR_ERR(prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) goto reset_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) st_map->progs[i] = prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) prog->aux->attach_btf_id != st_ops->type_id ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) prog->expected_attach_type != i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) goto reset_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) tprogs[BPF_TRAMP_FENTRY].progs[0] = prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) tprogs[BPF_TRAMP_FENTRY].nr_progs = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) flags = st_ops->func_models[i].ret_size > 0 ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) BPF_TRAMP_F_RET_FENTRY_RET : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) err = arch_prepare_bpf_trampoline(NULL, image,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) st_map->image + PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) &st_ops->func_models[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) flags, tprogs, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) goto reset_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) *(void **)(kdata + moff) = image;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) image += err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) /* put prog_id to udata */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) *(unsigned long *)(udata + moff) = prog->aux->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) refcount_set(&kvalue->refcnt, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) bpf_map_inc(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) set_memory_ro((long)st_map->image, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) trace_android_vh_set_memory_ro((unsigned long)st_map->image, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) set_memory_x((long)st_map->image, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) trace_android_vh_set_memory_x((unsigned long)st_map->image, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) err = st_ops->reg(kdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (likely(!err)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /* Pair with smp_load_acquire() during lookup_elem().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * It ensures the above udata updates (e.g. prog->aux->id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) smp_store_release(&kvalue->state, BPF_STRUCT_OPS_STATE_INUSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) /* Error during st_ops->reg(). It is very unlikely since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * the above init_member() should have caught it earlier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * before reg(). The only possibility is if there was a race
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * in registering the struct_ops (under the same name) to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) * a sub-system through different struct_ops's maps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) set_memory_nx((long)st_map->image, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) set_memory_rw((long)st_map->image, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) bpf_map_put(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) reset_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) bpf_struct_ops_map_put_progs(st_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) memset(uvalue, 0, map->value_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) memset(kvalue, 0, map->value_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) kfree(tprogs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) mutex_unlock(&st_map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) static int bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) enum bpf_struct_ops_state prev_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) struct bpf_struct_ops_map *st_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) st_map = (struct bpf_struct_ops_map *)map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) prev_state = cmpxchg(&st_map->kvalue.state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) BPF_STRUCT_OPS_STATE_INUSE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) BPF_STRUCT_OPS_STATE_TOBEFREE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) switch (prev_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) case BPF_STRUCT_OPS_STATE_INUSE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) st_map->st_ops->unreg(&st_map->kvalue.data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (refcount_dec_and_test(&st_map->kvalue.refcnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) bpf_map_put(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) case BPF_STRUCT_OPS_STATE_TOBEFREE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) return -EINPROGRESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) case BPF_STRUCT_OPS_STATE_INIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) /* Should never happen. Treat it as not found. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) void *value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (!value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) btf_type_seq_show(btf_vmlinux, map->btf_vmlinux_value_type_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) value, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) seq_puts(m, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) kfree(value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) static void bpf_struct_ops_map_free(struct bpf_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (st_map->progs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) bpf_struct_ops_map_put_progs(st_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) bpf_map_area_free(st_map->progs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) trace_android_vh_set_memory_rw((unsigned long)st_map->image, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) trace_android_vh_set_memory_nx((unsigned long)st_map->image, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) bpf_jit_free_exec(st_map->image);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) bpf_map_area_free(st_map->uvalue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) bpf_map_area_free(st_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) attr->map_flags || !attr->btf_vmlinux_value_type_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) const struct bpf_struct_ops *st_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) size_t map_total_size, st_map_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct bpf_struct_ops_map *st_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) const struct btf_type *t, *vt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) struct bpf_map_memory mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) struct bpf_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (!bpf_capable())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) return ERR_PTR(-EPERM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) st_ops = bpf_struct_ops_find_value(attr->btf_vmlinux_value_type_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) if (!st_ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) return ERR_PTR(-ENOTSUPP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) vt = st_ops->value_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) if (attr->value_size != vt->size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) t = st_ops->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) st_map_size = sizeof(*st_map) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) /* kvalue stores the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * struct bpf_struct_ops_tcp_congestions_ops
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) (vt->size - sizeof(struct bpf_struct_ops_value));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) map_total_size = st_map_size +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) /* uvalue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) sizeof(vt->size) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /* struct bpf_progs **progs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) btf_type_vlen(t) * sizeof(struct bpf_prog *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) err = bpf_map_charge_init(&mem, map_total_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) return ERR_PTR(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (!st_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) bpf_map_charge_finish(&mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) st_map->st_ops = st_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) map = &st_map->map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) st_map->progs =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_prog *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) st_map->image = bpf_jit_alloc_exec(PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (!st_map->uvalue || !st_map->progs || !st_map->image) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) bpf_struct_ops_map_free(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) bpf_map_charge_finish(&mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) mutex_init(&st_map->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) set_vm_flush_reset_perms(st_map->image);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) bpf_map_init_from_attr(map, attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) bpf_map_charge_move(&map->memory, &mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) static int bpf_struct_ops_map_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) const struct bpf_map_ops bpf_struct_ops_map_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) .map_alloc_check = bpf_struct_ops_map_alloc_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) .map_alloc = bpf_struct_ops_map_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) .map_free = bpf_struct_ops_map_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) .map_get_next_key = bpf_struct_ops_map_get_next_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) .map_lookup_elem = bpf_struct_ops_map_lookup_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) .map_delete_elem = bpf_struct_ops_map_delete_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) .map_update_elem = bpf_struct_ops_map_update_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) .map_seq_show_elem = bpf_struct_ops_map_seq_show_elem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) .map_btf_name = "bpf_struct_ops_map",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) .map_btf_id = &bpf_struct_ops_map_btf_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) /* "const void *" because some subsystem is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * passing a const (e.g. const struct tcp_congestion_ops *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) bool bpf_struct_ops_get(const void *kdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) struct bpf_struct_ops_value *kvalue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) return refcount_inc_not_zero(&kvalue->refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) void bpf_struct_ops_put(const void *kdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) struct bpf_struct_ops_value *kvalue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (refcount_dec_and_test(&kvalue->refcnt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) struct bpf_struct_ops_map *st_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) st_map = container_of(kvalue, struct bpf_struct_ops_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) kvalue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) bpf_map_put(&st_map->map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags