^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Copyright (c) 2020 Facebook */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/anon_inodes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/filter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) struct bpf_iter_target_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) const struct bpf_iter_reg *reg_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) u32 btf_id; /* cached value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) struct bpf_iter_link {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) struct bpf_link link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) struct bpf_iter_aux_info aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) struct bpf_iter_target_info *tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) struct bpf_iter_priv_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) struct bpf_iter_target_info *tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) const struct bpf_iter_seq_info *seq_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) struct bpf_prog *prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) u64 session_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) u64 seq_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) bool done_stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) u8 target_private[] __aligned(8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) static struct list_head targets = LIST_HEAD_INIT(targets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) static DEFINE_MUTEX(targets_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) /* protect bpf_iter_link changes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static DEFINE_MUTEX(link_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) /* incremented on every opened seq_file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static atomic64_t session_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) const struct bpf_iter_seq_info *seq_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static void bpf_iter_inc_seq_num(struct seq_file *seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct bpf_iter_priv_data *iter_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) target_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) iter_priv->seq_num++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) static void bpf_iter_dec_seq_num(struct seq_file *seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct bpf_iter_priv_data *iter_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) target_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) iter_priv->seq_num--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static void bpf_iter_done_stop(struct seq_file *seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct bpf_iter_priv_data *iter_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) target_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) iter_priv->done_stop = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /* maximum visited objects before bailing out */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define MAX_ITER_OBJECTS 1000000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* bpf_seq_read, a customized and simpler version for bpf iterator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * no_llseek is assumed for this file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * The following are differences from seq_read():
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * . fixed buffer size (PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * . assuming no_llseek
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * . stop() may call bpf program, handling potential overflow there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct seq_file *seq = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) size_t n, offs, copied = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int err = 0, num_objs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) mutex_lock(&seq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (!seq->buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) seq->size = PAGE_SIZE << 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) seq->buf = kvmalloc(seq->size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (!seq->buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) if (seq->count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) n = min(seq->count, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) err = copy_to_user(buf, seq->buf + seq->from, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) err = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) seq->count -= n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) seq->from += n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) copied = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) seq->from = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) p = seq->op->start(seq, &seq->index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) goto stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if (IS_ERR(p)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) err = PTR_ERR(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) seq->op->stop(seq, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) seq->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) err = seq->op->show(seq, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (err > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /* object is skipped, decrease seq_num, so next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * valid object can reuse the same seq_num.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) bpf_iter_dec_seq_num(seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) seq->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) } else if (err < 0 || seq_has_overflowed(seq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) err = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) seq->op->stop(seq, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) seq->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) loff_t pos = seq->index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) num_objs++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) offs = seq->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) p = seq->op->next(seq, p, &seq->index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) if (pos == seq->index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) pr_info_ratelimited("buggy seq_file .next function %ps "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) "did not updated position index\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) seq->op->next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) seq->index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (IS_ERR_OR_NULL(p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /* got a valid next object, increase seq_num */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) bpf_iter_inc_seq_num(seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (seq->count >= size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (num_objs >= MAX_ITER_OBJECTS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) if (offs == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) err = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) seq->op->stop(seq, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) err = seq->op->show(seq, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (err > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) bpf_iter_dec_seq_num(seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) seq->count = offs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) } else if (err < 0 || seq_has_overflowed(seq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) seq->count = offs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (offs == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) err = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) seq->op->stop(seq, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) stop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) offs = seq->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /* bpf program called if !p */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) seq->op->stop(seq, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) if (!p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (!seq_has_overflowed(seq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) bpf_iter_done_stop(seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) seq->count = offs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (offs == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) err = -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) n = min(seq->count, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) err = copy_to_user(buf, seq->buf, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) err = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) copied = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) seq->count -= n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) seq->from = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (!copied)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) copied = err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) *ppos += copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) mutex_unlock(&seq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) return copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static const struct bpf_iter_seq_info *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) __get_seq_info(struct bpf_iter_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) const struct bpf_iter_seq_info *seq_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (link->aux.map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) seq_info = link->aux.map->ops->iter_seq_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (seq_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return seq_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return link->tinfo->reg_info->seq_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static int iter_open(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct bpf_iter_link *link = inode->i_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return prepare_seq_file(file, link, __get_seq_info(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) static int iter_release(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) struct bpf_iter_priv_data *iter_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) struct seq_file *seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) seq = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (!seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) target_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (iter_priv->seq_info->fini_seq_private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) iter_priv->seq_info->fini_seq_private(seq->private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) bpf_prog_put(iter_priv->prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) seq->private = iter_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return seq_release_private(inode, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) const struct file_operations bpf_iter_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) .open = iter_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) .llseek = no_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) .read = bpf_seq_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) .release = iter_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /* The argument reg_info will be cached in bpf_iter_target_info.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * The common practice is to declare target reg_info as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * a const static variable and passed as an argument to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * bpf_iter_reg_target().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct bpf_iter_target_info *tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (!tinfo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) tinfo->reg_info = reg_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) INIT_LIST_HEAD(&tinfo->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) mutex_lock(&targets_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) list_add(&tinfo->list, &targets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) mutex_unlock(&targets_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) struct bpf_iter_target_info *tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) bool found = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) mutex_lock(&targets_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) list_for_each_entry(tinfo, &targets, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (reg_info == tinfo->reg_info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) list_del(&tinfo->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) kfree(tinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) found = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) mutex_unlock(&targets_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) WARN_ON(found == false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) static void cache_btf_id(struct bpf_iter_target_info *tinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) struct bpf_prog *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) tinfo->btf_id = prog->aux->attach_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) bool bpf_iter_prog_supported(struct bpf_prog *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) const char *attach_fname = prog->aux->attach_func_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) u32 prog_btf_id = prog->aux->attach_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) const char *prefix = BPF_ITER_FUNC_PREFIX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) struct bpf_iter_target_info *tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) int prefix_len = strlen(prefix);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) bool supported = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (strncmp(attach_fname, prefix, prefix_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) mutex_lock(&targets_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) list_for_each_entry(tinfo, &targets, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (tinfo->btf_id && tinfo->btf_id == prog_btf_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (!strcmp(attach_fname + prefix_len, tinfo->reg_info->target)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) cache_btf_id(tinfo, prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) mutex_unlock(&targets_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) if (supported) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) static void bpf_iter_link_release(struct bpf_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct bpf_iter_link *iter_link =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) container_of(link, struct bpf_iter_link, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (iter_link->tinfo->reg_info->detach_target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static void bpf_iter_link_dealloc(struct bpf_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) struct bpf_iter_link *iter_link =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) container_of(link, struct bpf_iter_link, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) kfree(iter_link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) static int bpf_iter_link_replace(struct bpf_link *link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct bpf_prog *new_prog,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) struct bpf_prog *old_prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) mutex_lock(&link_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) if (old_prog && link->prog != old_prog) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) ret = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (link->prog->type != new_prog->type ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) link->prog->expected_attach_type != new_prog->expected_attach_type ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) old_prog = xchg(&link->prog, new_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) bpf_prog_put(old_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) mutex_unlock(&link_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) struct seq_file *seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) struct bpf_iter_link *iter_link =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) container_of(link, struct bpf_iter_link, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) bpf_iter_show_fdinfo_t show_fdinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) seq_printf(seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) "target_name:\t%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) iter_link->tinfo->reg_info->target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) if (show_fdinfo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) show_fdinfo(&iter_link->aux, seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) struct bpf_link_info *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) struct bpf_iter_link *iter_link =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) container_of(link, struct bpf_iter_link, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) bpf_iter_fill_link_info_t fill_link_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) u32 ulen = info->iter.target_name_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) const char *target_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) u32 target_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (!ulen ^ !ubuf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) target_name = iter_link->tinfo->reg_info->target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) target_len = strlen(target_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) info->iter.target_name_len = target_len + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (ubuf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (ulen >= target_len + 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) if (copy_to_user(ubuf, target_name, target_len + 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) char zero = '\0';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (copy_to_user(ubuf, target_name, ulen - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) if (put_user(zero, ubuf + ulen - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (fill_link_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) return fill_link_info(&iter_link->aux, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) static const struct bpf_link_ops bpf_iter_link_lops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) .release = bpf_iter_link_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) .dealloc = bpf_iter_link_dealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) .update_prog = bpf_iter_link_replace,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) .show_fdinfo = bpf_iter_link_show_fdinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) .fill_link_info = bpf_iter_link_fill_link_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) bool bpf_link_is_iter(struct bpf_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) return link->ops == &bpf_iter_link_lops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) union bpf_iter_link_info __user *ulinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) struct bpf_link_primer link_primer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) struct bpf_iter_target_info *tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) union bpf_iter_link_info linfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct bpf_iter_link *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) u32 prog_btf_id, linfo_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) bool existed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (attr->link_create.target_fd || attr->link_create.flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) memset(&linfo, 0, sizeof(union bpf_iter_link_info));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) ulinfo = u64_to_user_ptr(attr->link_create.iter_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) linfo_len = attr->link_create.iter_info_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if (!ulinfo ^ !linfo_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) if (ulinfo) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) linfo_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) linfo_len = min_t(u32, linfo_len, sizeof(linfo));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (copy_from_user(&linfo, ulinfo, linfo_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) prog_btf_id = prog->aux->attach_btf_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) mutex_lock(&targets_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) list_for_each_entry(tinfo, &targets, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (tinfo->btf_id == prog_btf_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) existed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) mutex_unlock(&targets_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (!existed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (!link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) link->tinfo = tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) err = bpf_link_prime(&link->link, &link_primer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) kfree(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) if (tinfo->reg_info->attach_target) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) bpf_link_cleanup(&link_primer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) return bpf_link_settle(&link_primer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) struct bpf_iter_target_info *tinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) const struct bpf_iter_seq_info *seq_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) struct bpf_prog *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) priv_data->tinfo = tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) priv_data->seq_info = seq_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) priv_data->prog = prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) priv_data->session_id = atomic64_inc_return(&session_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) priv_data->seq_num = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) priv_data->done_stop = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) const struct bpf_iter_seq_info *seq_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct bpf_iter_priv_data *priv_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) struct bpf_iter_target_info *tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) struct bpf_prog *prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) u32 total_priv_dsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) struct seq_file *seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) mutex_lock(&link_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) prog = link->link.prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) bpf_prog_inc(prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) mutex_unlock(&link_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) tinfo = link->tinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) seq_info->seq_priv_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) priv_data = __seq_open_private(file, seq_info->seq_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) total_priv_dsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (!priv_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) goto release_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (seq_info->init_seq_private) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) goto release_seq_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) init_seq_meta(priv_data, tinfo, seq_info, prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) seq = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) seq->private = priv_data->target_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) release_seq_file:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) seq_release_private(file->f_inode, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) file->private_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) release_prog:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) bpf_prog_put(prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) int bpf_iter_new_fd(struct bpf_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) struct bpf_iter_link *iter_link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) struct file *file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) unsigned int flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) int err, fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (link->ops != &bpf_iter_link_lops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) flags = O_RDONLY | O_CLOEXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) fd = get_unused_fd_flags(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) if (fd < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (IS_ERR(file)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) err = PTR_ERR(file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) goto free_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) iter_link = container_of(link, struct bpf_iter_link, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) goto free_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) fd_install(fd, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) return fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) free_file:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) fput(file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) free_fd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) put_unused_fd(fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) struct bpf_iter_priv_data *iter_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) struct seq_file *seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) void *seq_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) seq = meta->seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) if (seq->file->f_op != &bpf_iter_fops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) seq_priv = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) target_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) if (in_stop && iter_priv->done_stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) meta->session_id = iter_priv->session_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) meta->seq_num = iter_priv->seq_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) return iter_priv->prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) migrate_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) ret = BPF_PROG_RUN(prog, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) migrate_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) /* bpf program can only return 0 or 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * 0 : okay
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) * 1 : retry the same object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) * The bpf_iter_run_prog() return value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * will be seq_ops->show() return value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) return ret == 0 ? 0 : -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags