^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Copyright(c) 2019 Intel Corporation. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/filter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) /* The BPF dispatcher is a multiway branch code generator. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * dispatcher is a mechanism to avoid the performance penalty of an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * indirect call, which is expensive when retpolines are enabled. A
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * dispatch client registers a BPF program into the dispatcher, and if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * there is available room in the dispatcher a direct call to the BPF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * program will be generated. All calls to the BPF programs called via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * the dispatcher will then be a direct call, instead of an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * indirect. The dispatcher hijacks a trampoline function it via the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * __fentry__ of the trampoline. The trampoline function has the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * following signature:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * unsigned int (*bpf_func)(const void *,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * const struct bpf_insn *));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) struct bpf_dispatcher *d, struct bpf_prog *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) if (prog == d->progs[i].prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) return &d->progs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct bpf_dispatcher *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) return bpf_dispatcher_find_prog(d, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct bpf_prog *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct bpf_dispatcher_prog *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) if (!prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) entry = bpf_dispatcher_find_prog(d, prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) if (entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) refcount_inc(&entry->users);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) entry = bpf_dispatcher_find_free(d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) if (!entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) bpf_prog_inc(prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) entry->prog = prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) refcount_set(&entry->users, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) d->num_progs++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct bpf_prog *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct bpf_dispatcher_prog *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (!prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) entry = bpf_dispatcher_find_prog(d, prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) if (!entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if (refcount_dec_and_test(&entry->users)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) entry->prog = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) bpf_prog_put(prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) d->num_progs--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) int __weak arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) if (d->progs[i].prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) void *old, *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) u32 noff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (!prev_num_progs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) old = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) noff = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) old = d->image + d->image_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) noff = d->image_off ^ (PAGE_SIZE / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) new = d->num_progs ? d->image + noff : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (bpf_dispatcher_prepare(d, new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (err || !new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) d->image_off = noff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct bpf_prog *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) bool changed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) int prev_num_progs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (from == to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) mutex_lock(&d->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (!d->image) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) d->image = bpf_jit_alloc_exec_page();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) if (!d->image)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) bpf_image_ksym_add(d->image, &d->ksym);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) prev_num_progs = d->num_progs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) changed |= bpf_dispatcher_remove_prog(d, from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) changed |= bpf_dispatcher_add_prog(d, to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) if (!changed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) bpf_dispatcher_update(d, prev_num_progs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) mutex_unlock(&d->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags