^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Robert Olsson <robert.olsson@its.uu.se> Uppsala Universitet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * & Swedish University of Agricultural Sciences.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Jens Laas <jens.laas@data.slu.se> Swedish University of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Agricultural Sciences.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Hans Liss <hans.liss@its.uu.se> Uppsala Universitet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * This work is based on the LPC-trie which is originally described in:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * An experimental study of compression methods for dynamic tries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Stefan Nilsson and Matti Tikkanen. Algorithmica, 33(1):19-33, 2002.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * https://www.csc.kth.se/~snilsson/software/dyntrie2/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * IP-address lookup using LC-tries. Stefan Nilsson and Gunnar Karlsson
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * IEEE Journal on Selected Areas in Communications, 17(6):1083-1092, June 1999
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * Code from fib_hash has been reused which includes the following header:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * INET An implementation of the TCP/IP protocol suite for the LINUX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * operating system. INET is implemented using the BSD Socket
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * interface as the means of communication with the user level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * IPv4 FIB: lookup engine and maintenance routines.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * Substantial contributions to this work comes from:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * David S. Miller, <davem@davemloft.net>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * Stephen Hemminger <shemminger@osdl.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * Paul E. McKenney <paulmck@us.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * Patrick McHardy <kaber@trash.net>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/cache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/socket.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/sockios.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/in.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/inet.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/inetdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/netdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/if_arp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/proc_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <linux/skbuff.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <linux/netlink.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #include <linux/notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include <net/net_namespace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include <net/ip.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include <net/protocol.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include <net/route.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #include <net/tcp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #include <net/sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #include <net/ip_fib.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #include <net/fib_notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #include <trace/events/fib.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #include "fib_lookup.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) static int call_fib_entry_notifier(struct notifier_block *nb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) enum fib_event_type event_type, u32 dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) int dst_len, struct fib_alias *fa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct fib_entry_notifier_info info = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) .info.extack = extack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) .dst = dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) .dst_len = dst_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) .fi = fa->fa_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) .tos = fa->fa_tos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) .type = fa->fa_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) .tb_id = fa->tb_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) return call_fib4_notifier(nb, event_type, &info.info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static int call_fib_entry_notifiers(struct net *net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) enum fib_event_type event_type, u32 dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) int dst_len, struct fib_alias *fa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) struct fib_entry_notifier_info info = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) .info.extack = extack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) .dst = dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) .dst_len = dst_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) .fi = fa->fa_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) .tos = fa->fa_tos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) .type = fa->fa_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) .tb_id = fa->tb_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return call_fib4_notifiers(net, event_type, &info.info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) #define MAX_STAT_DEPTH 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define KEYLENGTH (8*sizeof(t_key))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define KEY_MAX ((t_key)~0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) typedef unsigned int t_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #define IS_TRIE(n) ((n)->pos >= KEYLENGTH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define IS_TNODE(n) ((n)->bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define IS_LEAF(n) (!(n)->bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) struct key_vector {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) t_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) unsigned char pos; /* 2log(KEYLENGTH) bits needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) unsigned char bits; /* 2log(KEYLENGTH) bits needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) unsigned char slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /* This list pointer if valid if (pos | bits) == 0 (LEAF) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct hlist_head leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /* This array is valid if (pos | bits) > 0 (TNODE) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) struct key_vector __rcu *tnode[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct tnode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) t_key empty_children; /* KEYLENGTH bits needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) t_key full_children; /* KEYLENGTH bits needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct key_vector __rcu *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct key_vector kv[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define tn_bits kv[0].bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define TNODE_SIZE(n) offsetof(struct tnode, kv[0].tnode[n])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define LEAF_SIZE TNODE_SIZE(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct trie_use_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) unsigned int gets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) unsigned int backtrack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) unsigned int semantic_match_passed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) unsigned int semantic_match_miss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) unsigned int null_node_hit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) unsigned int resize_node_skipped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) struct trie_stat {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) unsigned int totdepth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) unsigned int maxdepth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) unsigned int tnodes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) unsigned int leaves;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) unsigned int nullpointers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) unsigned int prefixes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) unsigned int nodesizes[MAX_STAT_DEPTH];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct trie {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct key_vector kv[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct trie_use_stats __percpu *stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static struct key_vector *resize(struct trie *t, struct key_vector *tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static unsigned int tnode_free_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * synchronize_rcu after call_rcu for outstanding dirty memory; it should be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * especially useful before resizing the root node with PREEMPT_NONE configs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * the value was obtained experimentally, aiming to avoid visible slowdown.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) unsigned int sysctl_fib_sync_mem = 512 * 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) unsigned int sysctl_fib_sync_mem_min = 64 * 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) unsigned int sysctl_fib_sync_mem_max = 64 * 1024 * 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static struct kmem_cache *fn_alias_kmem __ro_after_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) static struct kmem_cache *trie_leaf_kmem __ro_after_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) static inline struct tnode *tn_info(struct key_vector *kv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return container_of(kv, struct tnode, kv[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /* caller must hold RTNL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #define node_parent(tn) rtnl_dereference(tn_info(tn)->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #define get_child(tn, i) rtnl_dereference((tn)->tnode[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /* caller must hold RCU read lock or RTNL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #define node_parent_rcu(tn) rcu_dereference_rtnl(tn_info(tn)->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) #define get_child_rcu(tn, i) rcu_dereference_rtnl((tn)->tnode[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) /* wrapper for rcu_assign_pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static inline void node_set_parent(struct key_vector *n, struct key_vector *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) rcu_assign_pointer(tn_info(n)->parent, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) #define NODE_INIT_PARENT(n, p) RCU_INIT_POINTER(tn_info(n)->parent, p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) /* This provides us with the number of children in this node, in the case of a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * leaf this will return 0 meaning none of the children are accessible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static inline unsigned long child_length(const struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return (1ul << tn->bits) & ~(1ul);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) #define get_cindex(key, kv) (((key) ^ (kv)->key) >> (kv)->pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static inline unsigned long get_index(t_key key, struct key_vector *kv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) unsigned long index = key ^ kv->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if ((BITS_PER_LONG <= KEYLENGTH) && (KEYLENGTH == kv->pos))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return index >> kv->pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /* To understand this stuff, an understanding of keys and all their bits is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * necessary. Every node in the trie has a key associated with it, but not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * all of the bits in that key are significant.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * Consider a node 'n' and its parent 'tp'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * If n is a leaf, every bit in its key is significant. Its presence is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * necessitated by path compression, since during a tree traversal (when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * searching for a leaf - unless we are doing an insertion) we will completely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * ignore all skipped bits we encounter. Thus we need to verify, at the end of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * a potentially successful search, that we have indeed been walking the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * correct key path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * Note that we can never "miss" the correct key in the tree if present by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * following the wrong path. Path compression ensures that segments of the key
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * that are the same for all keys with a given prefix are skipped, but the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * skipped part *is* identical for each node in the subtrie below the skipped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * bit! trie_insert() in this implementation takes care of that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * if n is an internal node - a 'tnode' here, the various parts of its key
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * have many different meanings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * Example:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * _________________________________________________________________
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * | i | i | i | i | i | i | i | N | N | N | S | S | S | S | S | C |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * -----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * _________________________________________________________________
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * | C | C | C | u | u | u | u | u | u | u | u | u | u | u | u | u |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * -----------------------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * tp->pos = 22
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * tp->bits = 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * n->pos = 13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * n->bits = 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * First, let's just ignore the bits that come before the parent tp, that is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * the bits from (tp->pos + tp->bits) to 31. They are *known* but at this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * point we do not use them for anything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * The bits from (tp->pos) to (tp->pos + tp->bits - 1) - "N", above - are the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * index into the parent's child array. That is, they will be used to find
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * 'n' among tp's children.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * The bits from (n->pos + n->bits) to (tp->pos - 1) - "S" - are skipped bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * for the node n.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * All the bits we have seen so far are significant to the node n. The rest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * of the bits are really not needed or indeed known in n->key.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * The bits from (n->pos) to (n->pos + n->bits - 1) - "C" - are the index into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * n's child array, and will of course be different for each child.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) * The rest of the bits, from 0 to (n->pos -1) - "u" - are completely unknown
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * at this point.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static const int halve_threshold = 25;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static const int inflate_threshold = 50;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static const int halve_threshold_root = 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) static const int inflate_threshold_root = 30;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static void __alias_free_mem(struct rcu_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct fib_alias *fa = container_of(head, struct fib_alias, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) kmem_cache_free(fn_alias_kmem, fa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) static inline void alias_free_mem_rcu(struct fib_alias *fa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) call_rcu(&fa->rcu, __alias_free_mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) #define TNODE_VMALLOC_MAX \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) ilog2((SIZE_MAX - TNODE_SIZE(0)) / sizeof(struct key_vector *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) static void __node_free_rcu(struct rcu_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) struct tnode *n = container_of(head, struct tnode, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (!n->tn_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) kmem_cache_free(trie_leaf_kmem, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) kvfree(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) #define node_free(n) call_rcu(&tn_info(n)->rcu, __node_free_rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static struct tnode *tnode_alloc(int bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /* verify bits is within bounds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (bits > TNODE_VMALLOC_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) /* determine size and verify it is non-zero and didn't overflow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) size = TNODE_SIZE(1ul << bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (size <= PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return kzalloc(size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return vzalloc(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) static inline void empty_child_inc(struct key_vector *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) tn_info(n)->empty_children++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (!tn_info(n)->empty_children)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) tn_info(n)->full_children++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) static inline void empty_child_dec(struct key_vector *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) if (!tn_info(n)->empty_children)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) tn_info(n)->full_children--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) tn_info(n)->empty_children--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static struct key_vector *leaf_new(t_key key, struct fib_alias *fa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) struct key_vector *l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) struct tnode *kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) kv = kmem_cache_alloc(trie_leaf_kmem, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (!kv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /* initialize key vector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) l = kv->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) l->key = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) l->pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) l->bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) l->slen = fa->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) /* link leaf to fib alias */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) INIT_HLIST_HEAD(&l->leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) hlist_add_head(&fa->fa_list, &l->leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) static struct key_vector *tnode_new(t_key key, int pos, int bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) unsigned int shift = pos + bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) struct key_vector *tn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) struct tnode *tnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) /* verify bits and pos their msb bits clear and values are valid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) BUG_ON(!bits || (shift > KEYLENGTH));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) tnode = tnode_alloc(bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (!tnode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) pr_debug("AT %p s=%zu %zu\n", tnode, TNODE_SIZE(0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) sizeof(struct key_vector *) << bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (bits == KEYLENGTH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) tnode->full_children = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) tnode->empty_children = 1ul << bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) tn = tnode->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) tn->key = (shift < KEYLENGTH) ? (key >> shift) << shift : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) tn->pos = pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) tn->bits = bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) tn->slen = pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) return tn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /* Check whether a tnode 'n' is "full", i.e. it is an internal node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * and no bits are skipped. See discussion in dyntree paper p. 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) static inline int tnode_full(struct key_vector *tn, struct key_vector *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) return n && ((n->pos + n->bits) == tn->pos) && IS_TNODE(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) /* Add a child at position i overwriting the old value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * Update the value of full_children and empty_children.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) static void put_child(struct key_vector *tn, unsigned long i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) struct key_vector *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) struct key_vector *chi = get_child(tn, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) int isfull, wasfull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) BUG_ON(i >= child_length(tn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /* update emptyChildren, overflow into fullChildren */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (!n && chi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) empty_child_inc(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (n && !chi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) empty_child_dec(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) /* update fullChildren */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) wasfull = tnode_full(tn, chi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) isfull = tnode_full(tn, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (wasfull && !isfull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) tn_info(tn)->full_children--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) else if (!wasfull && isfull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) tn_info(tn)->full_children++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (n && (tn->slen < n->slen))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) tn->slen = n->slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) rcu_assign_pointer(tn->tnode[i], n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) static void update_children(struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) /* update all of the child parent pointers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) for (i = child_length(tn); i;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) struct key_vector *inode = get_child(tn, --i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (!inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /* Either update the children of a tnode that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * already belongs to us or update the child
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * to point to ourselves.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (node_parent(inode) == tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) update_children(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) node_set_parent(inode, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) static inline void put_child_root(struct key_vector *tp, t_key key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct key_vector *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (IS_TRIE(tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) rcu_assign_pointer(tp->tnode[0], n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) put_child(tp, get_index(key, tp), n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) static inline void tnode_free_init(struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) tn_info(tn)->rcu.next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) static inline void tnode_free_append(struct key_vector *tn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) struct key_vector *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) tn_info(n)->rcu.next = tn_info(tn)->rcu.next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) tn_info(tn)->rcu.next = &tn_info(n)->rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) static void tnode_free(struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) struct callback_head *head = &tn_info(tn)->rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) while (head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) head = head->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) tnode_free_size += TNODE_SIZE(1ul << tn->bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) node_free(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) tn = container_of(head, struct tnode, rcu)->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (tnode_free_size >= sysctl_fib_sync_mem) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) tnode_free_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) static struct key_vector *replace(struct trie *t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) struct key_vector *oldtnode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) struct key_vector *tp = node_parent(oldtnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) /* setup the parent pointer out of and back into this node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) NODE_INIT_PARENT(tn, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) put_child_root(tp, tn->key, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /* update all of the child parent pointers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) update_children(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /* all pointers should be clean so we are done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) tnode_free(oldtnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) /* resize children now that oldtnode is freed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) for (i = child_length(tn); i;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) struct key_vector *inode = get_child(tn, --i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) /* resize child node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) if (tnode_full(tn, inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) tn = resize(t, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) return tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) static struct key_vector *inflate(struct trie *t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) struct key_vector *oldtnode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) struct key_vector *tn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) t_key m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) pr_debug("In inflate\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) tn = tnode_new(oldtnode->key, oldtnode->pos - 1, oldtnode->bits + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (!tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) goto notnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) /* prepare oldtnode to be freed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) tnode_free_init(oldtnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) /* Assemble all of the pointers in our cluster, in this case that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * represents all of the pointers out of our allocated nodes that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * point to existing tnodes and the links between our allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) for (i = child_length(oldtnode), m = 1u << tn->pos; i;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) struct key_vector *inode = get_child(oldtnode, --i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) struct key_vector *node0, *node1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) unsigned long j, k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /* An empty child */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (!inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) /* A leaf or an internal node with skipped bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (!tnode_full(oldtnode, inode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) put_child(tn, get_index(inode->key, tn), inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) /* drop the node in the old tnode free list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) tnode_free_append(oldtnode, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) /* An internal node with two children */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) if (inode->bits == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) put_child(tn, 2 * i + 1, get_child(inode, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) put_child(tn, 2 * i, get_child(inode, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) /* We will replace this node 'inode' with two new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * ones, 'node0' and 'node1', each with half of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * original children. The two new nodes will have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) * a position one bit further down the key and this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) * means that the "significant" part of their keys
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) * (see the discussion near the top of this file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * will differ by one bit, which will be "0" in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * node0's key and "1" in node1's key. Since we are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) * moving the key position by one step, the bit that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) * we are moving away from - the bit at position
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * (tn->pos) - is the one that will differ between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * node0 and node1. So... we synthesize that bit in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * two new keys.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) node1 = tnode_new(inode->key | m, inode->pos, inode->bits - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) if (!node1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) goto nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) node0 = tnode_new(inode->key, inode->pos, inode->bits - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) tnode_free_append(tn, node1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (!node0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) goto nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) tnode_free_append(tn, node0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) /* populate child pointers in new nodes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) for (k = child_length(inode), j = k / 2; j;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) put_child(node1, --j, get_child(inode, --k));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) put_child(node0, j, get_child(inode, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) put_child(node1, --j, get_child(inode, --k));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) put_child(node0, j, get_child(inode, j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) /* link new nodes to parent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) NODE_INIT_PARENT(node1, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) NODE_INIT_PARENT(node0, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) /* link parent to nodes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) put_child(tn, 2 * i + 1, node1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) put_child(tn, 2 * i, node0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) /* setup the parent pointers into and out of this node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) return replace(t, oldtnode, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) nomem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) /* all pointers should be clean so we are done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) tnode_free(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) notnode:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) static struct key_vector *halve(struct trie *t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) struct key_vector *oldtnode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) struct key_vector *tn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) pr_debug("In halve\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) tn = tnode_new(oldtnode->key, oldtnode->pos + 1, oldtnode->bits - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) if (!tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) goto notnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) /* prepare oldtnode to be freed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) tnode_free_init(oldtnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) /* Assemble all of the pointers in our cluster, in this case that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) * represents all of the pointers out of our allocated nodes that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * point to existing tnodes and the links between our allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) * nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) for (i = child_length(oldtnode); i;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) struct key_vector *node1 = get_child(oldtnode, --i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) struct key_vector *node0 = get_child(oldtnode, --i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) struct key_vector *inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) /* At least one of the children is empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) if (!node1 || !node0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) put_child(tn, i / 2, node1 ? : node0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) /* Two nonempty children */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) inode = tnode_new(node0->key, oldtnode->pos, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) if (!inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) goto nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) tnode_free_append(tn, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) /* initialize pointers out of node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) put_child(inode, 1, node1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) put_child(inode, 0, node0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) NODE_INIT_PARENT(inode, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) /* link parent to node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) put_child(tn, i / 2, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) /* setup the parent pointers into and out of this node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) return replace(t, oldtnode, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) nomem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) /* all pointers should be clean so we are done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) tnode_free(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) notnode:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) static struct key_vector *collapse(struct trie *t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) struct key_vector *oldtnode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) struct key_vector *n, *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) /* scan the tnode looking for that one child that might still exist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) for (n = NULL, i = child_length(oldtnode); !n && i;)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) n = get_child(oldtnode, --i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) /* compress one level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) tp = node_parent(oldtnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) put_child_root(tp, oldtnode->key, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) node_set_parent(n, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) /* drop dead node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) node_free(oldtnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) return tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) static unsigned char update_suffix(struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) unsigned char slen = tn->pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) unsigned long stride, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) unsigned char slen_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) /* only vector 0 can have a suffix length greater than or equal to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) * tn->pos + tn->bits, the second highest node will have a suffix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * length at most of tn->pos + tn->bits - 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) slen_max = min_t(unsigned char, tn->pos + tn->bits - 1, tn->slen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) /* search though the list of children looking for nodes that might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * have a suffix greater than the one we currently have. This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) * why we start with a stride of 2 since a stride of 1 would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * represent the nodes with suffix length equal to tn->pos
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) for (i = 0, stride = 0x2ul ; i < child_length(tn); i += stride) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) struct key_vector *n = get_child(tn, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (!n || (n->slen <= slen))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) /* update stride and slen based on new value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) stride <<= (n->slen - slen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) slen = n->slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) i &= ~(stride - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) /* stop searching if we have hit the maximum possible value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (slen >= slen_max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) tn->slen = slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) return slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) /* From "Implementing a dynamic compressed trie" by Stefan Nilsson of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * the Helsinki University of Technology and Matti Tikkanen of Nokia
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * Telecommunications, page 6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * "A node is doubled if the ratio of non-empty children to all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * children in the *doubled* node is at least 'high'."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) * 'high' in this instance is the variable 'inflate_threshold'. It
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * is expressed as a percentage, so we multiply it with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * child_length() and instead of multiplying by 2 (since the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * child array will be doubled by inflate()) and multiplying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * the left-hand side by 100 (to handle the percentage thing) we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * multiply the left-hand side by 50.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * The left-hand side may look a bit weird: child_length(tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * - tn->empty_children is of course the number of non-null children
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * in the current node. tn->full_children is the number of "full"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * children, that is non-null tnodes with a skip value of 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * All of those will be doubled in the resulting inflated tnode, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * we just count them one extra time here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * A clearer way to write this would be:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * to_be_doubled = tn->full_children;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * not_to_be_doubled = child_length(tn) - tn->empty_children -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * tn->full_children;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * new_child_length = child_length(tn) * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) * new_fill_factor = 100 * (not_to_be_doubled + 2*to_be_doubled) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) * new_child_length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * if (new_fill_factor >= inflate_threshold)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) * ...and so on, tho it would mess up the while () loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * anyway,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * 100 * (not_to_be_doubled + 2*to_be_doubled) / new_child_length >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * inflate_threshold
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * avoid a division:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * 100 * (not_to_be_doubled + 2*to_be_doubled) >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) * inflate_threshold * new_child_length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) * expand not_to_be_doubled and to_be_doubled, and shorten:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * 100 * (child_length(tn) - tn->empty_children +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * tn->full_children) >= inflate_threshold * new_child_length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) * expand new_child_length:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) * 100 * (child_length(tn) - tn->empty_children +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * tn->full_children) >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) * inflate_threshold * child_length(tn) * 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * shorten again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) * 50 * (tn->full_children + child_length(tn) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * tn->empty_children) >= inflate_threshold *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * child_length(tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) static inline bool should_inflate(struct key_vector *tp, struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) unsigned long used = child_length(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) unsigned long threshold = used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) /* Keep root node larger */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) threshold *= IS_TRIE(tp) ? inflate_threshold_root : inflate_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) used -= tn_info(tn)->empty_children;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) used += tn_info(tn)->full_children;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) /* if bits == KEYLENGTH then pos = 0, and will fail below */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) return (used > 1) && tn->pos && ((50 * used) >= threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) static inline bool should_halve(struct key_vector *tp, struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) unsigned long used = child_length(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) unsigned long threshold = used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) /* Keep root node larger */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) threshold *= IS_TRIE(tp) ? halve_threshold_root : halve_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) used -= tn_info(tn)->empty_children;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) /* if bits == KEYLENGTH then used = 100% on wrap, and will fail below */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) return (used > 1) && (tn->bits > 1) && ((100 * used) < threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) static inline bool should_collapse(struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) unsigned long used = child_length(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) used -= tn_info(tn)->empty_children;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) /* account for bits == KEYLENGTH case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) if ((tn->bits == KEYLENGTH) && tn_info(tn)->full_children)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) used -= KEY_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) /* One child or none, time to drop us from the trie */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) return used < 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) #define MAX_WORK 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) static struct key_vector *resize(struct trie *t, struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) struct trie_use_stats __percpu *stats = t->stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) struct key_vector *tp = node_parent(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) unsigned long cindex = get_index(tn->key, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) int max_work = MAX_WORK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) pr_debug("In tnode_resize %p inflate_threshold=%d threshold=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) tn, inflate_threshold, halve_threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) /* track the tnode via the pointer from the parent instead of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) * doing it ourselves. This way we can let RCU fully do its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * thing without us interfering
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) BUG_ON(tn != get_child(tp, cindex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) /* Double as long as the resulting node has a number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * nonempty nodes that are above the threshold.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) while (should_inflate(tp, tn) && max_work) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) tp = inflate(t, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) if (!tp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) this_cpu_inc(stats->resize_node_skipped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) max_work--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) tn = get_child(tp, cindex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) /* update parent in case inflate failed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) tp = node_parent(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) /* Return if at least one inflate is run */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) if (max_work != MAX_WORK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) return tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) /* Halve as long as the number of empty children in this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * node is above threshold.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) while (should_halve(tp, tn) && max_work) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) tp = halve(t, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) if (!tp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) this_cpu_inc(stats->resize_node_skipped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) max_work--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) tn = get_child(tp, cindex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) /* Only one child remains */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) if (should_collapse(tn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) return collapse(t, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) /* update parent in case halve failed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) return node_parent(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) static void node_pull_suffix(struct key_vector *tn, unsigned char slen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) unsigned char node_slen = tn->slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) while ((node_slen > tn->pos) && (node_slen > slen)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) slen = update_suffix(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) if (node_slen == slen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) tn = node_parent(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) node_slen = tn->slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) static void node_push_suffix(struct key_vector *tn, unsigned char slen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) while (tn->slen < slen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) tn->slen = slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) tn = node_parent(tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) /* rcu_read_lock needs to be hold by caller from readside */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) static struct key_vector *fib_find_node(struct trie *t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) struct key_vector **tp, u32 key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) struct key_vector *pn, *n = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) unsigned long index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) pn = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) n = get_child_rcu(n, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) index = get_cindex(key, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) /* This bit of code is a bit tricky but it combines multiple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * checks into a single check. The prefix consists of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) * prefix plus zeros for the bits in the cindex. The index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * is the difference between the key and this value. From
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * this we can actually derive several pieces of data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) * if (index >= (1ul << bits))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) * we have a mismatch in skip bits and failed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) * else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) * we know the value is cindex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * This check is safe even if bits == KEYLENGTH due to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * fact that we can only allocate a node with 32 bits if a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) * long is greater than 32 bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) if (index >= (1ul << n->bits)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) n = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) /* keep searching until we find a perfect match leaf or NULL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) } while (IS_TNODE(n));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) *tp = pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) /* Return the first fib alias matching TOS with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * priority less than or equal to PRIO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) * If 'find_first' is set, return the first matching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) * fib alias, regardless of TOS and priority.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) static struct fib_alias *fib_find_alias(struct hlist_head *fah, u8 slen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) u8 tos, u32 prio, u32 tb_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) bool find_first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) if (!fah)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) hlist_for_each_entry(fa, fah, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) if (fa->fa_slen < slen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) if (fa->fa_slen != slen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) if (fa->tb_id > tb_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) if (fa->tb_id != tb_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) if (find_first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) return fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) if (fa->fa_tos > tos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) if (fa->fa_info->fib_priority >= prio || fa->fa_tos < tos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) return fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) static struct fib_alias *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) fib_find_matching_alias(struct net *net, const struct fib_rt_info *fri)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) u8 slen = KEYLENGTH - fri->dst_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) struct key_vector *l, *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) struct fib_table *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) struct trie *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) tb = fib_get_table(net, fri->tb_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) if (!tb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) l = fib_find_node(t, &tp, be32_to_cpu(fri->dst));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (!l)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) hlist_for_each_entry_rcu(fa, &l->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) if (fa->fa_slen == slen && fa->tb_id == fri->tb_id &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) fa->fa_tos == fri->tos && fa->fa_info == fri->fi &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) fa->fa_type == fri->type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) return fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) void fib_alias_hw_flags_set(struct net *net, const struct fib_rt_info *fri)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) struct fib_alias *fa_match;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) fa_match = fib_find_matching_alias(net, fri);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) if (!fa_match)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) fa_match->offload = fri->offload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) fa_match->trap = fri->trap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) EXPORT_SYMBOL_GPL(fib_alias_hw_flags_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) static void trie_rebalance(struct trie *t, struct key_vector *tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) while (!IS_TRIE(tn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) tn = resize(t, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) static int fib_insert_node(struct trie *t, struct key_vector *tp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) struct fib_alias *new, t_key key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) struct key_vector *n, *l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) l = leaf_new(key, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) if (!l)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) goto noleaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) /* retrieve child from parent node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) n = get_child(tp, get_index(key, tp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) /* Case 2: n is a LEAF or a TNODE and the key doesn't match.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) * Add a new tnode here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * first tnode need some special handling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) * leaves us in position for handling as case 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if (n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) struct key_vector *tn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) tn = tnode_new(key, __fls(key ^ n->key), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) if (!tn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) goto notnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) /* initialize routes out of node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) NODE_INIT_PARENT(tn, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) put_child(tn, get_index(key, tn) ^ 1, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) /* start adding routes into the node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) put_child_root(tp, key, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) node_set_parent(n, tn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) /* parent now has a NULL spot where the leaf can go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) tp = tn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) /* Case 3: n is NULL, and will just insert a new leaf */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) node_push_suffix(tp, new->fa_slen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) NODE_INIT_PARENT(l, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) put_child_root(tp, key, l);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) trie_rebalance(t, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) notnode:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) node_free(l);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) noleaf:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) static int fib_insert_alias(struct trie *t, struct key_vector *tp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) struct key_vector *l, struct fib_alias *new,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) struct fib_alias *fa, t_key key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) if (!l)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) return fib_insert_node(t, tp, new, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) if (fa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) hlist_add_before_rcu(&new->fa_list, &fa->fa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) struct fib_alias *last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) hlist_for_each_entry(last, &l->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) if (new->fa_slen < last->fa_slen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) if ((new->fa_slen == last->fa_slen) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) (new->tb_id > last->tb_id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) fa = last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) if (fa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) hlist_add_behind_rcu(&new->fa_list, &fa->fa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) hlist_add_head_rcu(&new->fa_list, &l->leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) /* if we added to the tail node then we need to update slen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) if (l->slen < new->fa_slen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) l->slen = new->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) node_push_suffix(tp, new->fa_slen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) static bool fib_valid_key_len(u32 key, u8 plen, struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) if (plen > KEYLENGTH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) NL_SET_ERR_MSG(extack, "Invalid prefix length");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) if ((plen < KEYLENGTH) && (key << plen)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) NL_SET_ERR_MSG(extack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) "Invalid prefix for given prefix length");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) static void fib_remove_alias(struct trie *t, struct key_vector *tp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) struct key_vector *l, struct fib_alias *old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) /* Caller must hold RTNL. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) int fib_table_insert(struct net *net, struct fib_table *tb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) struct fib_config *cfg, struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) struct trie *t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) struct fib_alias *fa, *new_fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) struct key_vector *l, *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) u16 nlflags = NLM_F_EXCL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) struct fib_info *fi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) u8 plen = cfg->fc_dst_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) u8 slen = KEYLENGTH - plen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) u8 tos = cfg->fc_tos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) u32 key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) key = ntohl(cfg->fc_dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) if (!fib_valid_key_len(key, plen, extack))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) pr_debug("Insert table=%u %08x/%d\n", tb->tb_id, key, plen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) fi = fib_create_info(cfg, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) if (IS_ERR(fi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) err = PTR_ERR(fi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) l = fib_find_node(t, &tp, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) fa = l ? fib_find_alias(&l->leaf, slen, tos, fi->fib_priority,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) tb->tb_id, false) : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) /* Now fa, if non-NULL, points to the first fib alias
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * with the same keys [prefix,tos,priority], if such key already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * exists or to the node before which we will insert new one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * If fa is NULL, we will need to allocate a new one and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * insert to the tail of the section matching the suffix length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) * of the new alias.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) if (fa && fa->fa_tos == tos &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) fa->fa_info->fib_priority == fi->fib_priority) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) struct fib_alias *fa_first, *fa_match;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) err = -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (cfg->fc_nlflags & NLM_F_EXCL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) nlflags &= ~NLM_F_EXCL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) /* We have 2 goals:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * 1. Find exact match for type, scope, fib_info to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * duplicate routes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) * 2. Find next 'fa' (or head), NLM_F_APPEND inserts before it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) fa_match = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) fa_first = fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) hlist_for_each_entry_from(fa, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) if ((fa->fa_slen != slen) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) (fa->tb_id != tb->tb_id) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) (fa->fa_tos != tos))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) if (fa->fa_info->fib_priority != fi->fib_priority)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) if (fa->fa_type == cfg->fc_type &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) fa->fa_info == fi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) fa_match = fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) if (cfg->fc_nlflags & NLM_F_REPLACE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) struct fib_info *fi_drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) u8 state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) nlflags |= NLM_F_REPLACE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) fa = fa_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) if (fa_match) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) if (fa == fa_match)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) err = -ENOBUFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) new_fa = kmem_cache_alloc(fn_alias_kmem, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) if (!new_fa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) fi_drop = fa->fa_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) new_fa->fa_tos = fa->fa_tos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) new_fa->fa_info = fi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) new_fa->fa_type = cfg->fc_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) state = fa->fa_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) new_fa->fa_state = state & ~FA_S_ACCESSED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) new_fa->fa_slen = fa->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) new_fa->tb_id = tb->tb_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) new_fa->fa_default = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) new_fa->offload = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) new_fa->trap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) hlist_replace_rcu(&fa->fa_list, &new_fa->fa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) if (fib_find_alias(&l->leaf, fa->fa_slen, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) tb->tb_id, true) == new_fa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) enum fib_event_type fib_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) fib_event = FIB_EVENT_ENTRY_REPLACE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) err = call_fib_entry_notifiers(net, fib_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) key, plen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) new_fa, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) hlist_replace_rcu(&new_fa->fa_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) &fa->fa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) goto out_free_new_fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) rtmsg_fib(RTM_NEWROUTE, htonl(key), new_fa, plen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) tb->tb_id, &cfg->fc_nlinfo, nlflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) alias_free_mem_rcu(fa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) fib_release_info(fi_drop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) if (state & FA_S_ACCESSED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) rt_cache_flush(cfg->fc_nlinfo.nl_net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) goto succeeded;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) /* Error if we find a perfect match which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * uses the same scope, type, and nexthop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) * information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) if (fa_match)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) if (cfg->fc_nlflags & NLM_F_APPEND)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) nlflags |= NLM_F_APPEND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) fa = fa_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) err = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) if (!(cfg->fc_nlflags & NLM_F_CREATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) nlflags |= NLM_F_CREATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) err = -ENOBUFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) new_fa = kmem_cache_alloc(fn_alias_kmem, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) if (!new_fa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) new_fa->fa_info = fi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) new_fa->fa_tos = tos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) new_fa->fa_type = cfg->fc_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) new_fa->fa_state = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) new_fa->fa_slen = slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) new_fa->tb_id = tb->tb_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) new_fa->fa_default = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) new_fa->offload = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) new_fa->trap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) /* Insert new entry to the list. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) err = fib_insert_alias(t, tp, l, new_fa, fa, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) goto out_free_new_fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) /* The alias was already inserted, so the node must exist. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) l = l ? l : fib_find_node(t, &tp, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) if (WARN_ON_ONCE(!l))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) goto out_free_new_fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) if (fib_find_alias(&l->leaf, new_fa->fa_slen, 0, 0, tb->tb_id, true) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) new_fa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) enum fib_event_type fib_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) fib_event = FIB_EVENT_ENTRY_REPLACE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) err = call_fib_entry_notifiers(net, fib_event, key, plen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) new_fa, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) goto out_remove_new_fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) if (!plen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) tb->tb_num_default++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) rt_cache_flush(cfg->fc_nlinfo.nl_net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) rtmsg_fib(RTM_NEWROUTE, htonl(key), new_fa, plen, new_fa->tb_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) &cfg->fc_nlinfo, nlflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) succeeded:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) out_remove_new_fa:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) fib_remove_alias(t, tp, l, new_fa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) out_free_new_fa:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) kmem_cache_free(fn_alias_kmem, new_fa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) fib_release_info(fi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) static inline t_key prefix_mismatch(t_key key, struct key_vector *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) t_key prefix = n->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) return (key ^ prefix) & (prefix | -prefix);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) bool fib_lookup_good_nhc(const struct fib_nh_common *nhc, int fib_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) const struct flowi4 *flp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) if (nhc->nhc_flags & RTNH_F_DEAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) if (ip_ignore_linkdown(nhc->nhc_dev) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) nhc->nhc_flags & RTNH_F_LINKDOWN &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) !(fib_flags & FIB_LOOKUP_IGNORE_LINKSTATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) if (!(flp->flowi4_flags & FLOWI_FLAG_SKIP_NH_OIF)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) if (flp->flowi4_oif &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) flp->flowi4_oif != nhc->nhc_oif)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) /* should be called with rcu_read_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) int fib_table_lookup(struct fib_table *tb, const struct flowi4 *flp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) struct fib_result *res, int fib_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) struct trie *t = (struct trie *) tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) struct trie_use_stats __percpu *stats = t->stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) const t_key key = ntohl(flp->daddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) struct key_vector *n, *pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) unsigned long index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) t_key cindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) pn = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) cindex = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) n = get_child_rcu(pn, cindex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) if (!n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) trace_fib_table_lookup(tb->tb_id, flp, NULL, -EAGAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) this_cpu_inc(stats->gets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) /* Step 1: Travel to the longest prefix match in the trie */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) index = get_cindex(key, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) /* This bit of code is a bit tricky but it combines multiple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) * checks into a single check. The prefix consists of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) * prefix plus zeros for the "bits" in the prefix. The index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) * is the difference between the key and this value. From
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) * this we can actually derive several pieces of data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) * if (index >= (1ul << bits))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) * we have a mismatch in skip bits and failed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) * else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) * we know the value is cindex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) * This check is safe even if bits == KEYLENGTH due to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) * fact that we can only allocate a node with 32 bits if a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) * long is greater than 32 bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) if (index >= (1ul << n->bits))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) /* we have found a leaf. Prefixes have already been compared */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) if (IS_LEAF(n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) /* only record pn and cindex if we are going to be chopping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) * bits later. Otherwise we are just wasting cycles.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) if (n->slen > n->pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) pn = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) cindex = index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) n = get_child_rcu(n, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) if (unlikely(!n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) goto backtrace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) /* Step 2: Sort out leaves and begin backtracing for longest prefix */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) /* record the pointer where our next node pointer is stored */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) struct key_vector __rcu **cptr = n->tnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) /* This test verifies that none of the bits that differ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) * between the key and the prefix exist in the region of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) * the lsb and higher in the prefix.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) if (unlikely(prefix_mismatch(key, n)) || (n->slen == n->pos))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) goto backtrace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) /* exit out and process leaf */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) if (unlikely(IS_LEAF(n)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) /* Don't bother recording parent info. Since we are in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) * prefix match mode we will have to come back to wherever
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) * we started this traversal anyway
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) while ((n = rcu_dereference(*cptr)) == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) backtrace:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) this_cpu_inc(stats->null_node_hit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) /* If we are at cindex 0 there are no more bits for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) * us to strip at this level so we must ascend back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) * up one level to see if there are any more bits to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) * be stripped there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) while (!cindex) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) t_key pkey = pn->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) /* If we don't have a parent then there is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) * nothing for us to do as we do not have any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) * further nodes to parse.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) if (IS_TRIE(pn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) trace_fib_table_lookup(tb->tb_id, flp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) NULL, -EAGAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) this_cpu_inc(stats->backtrack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) /* Get Child's index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) pn = node_parent_rcu(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) cindex = get_index(pkey, pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) /* strip the least significant bit from the cindex */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) cindex &= cindex - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) /* grab pointer for next child node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) cptr = &pn->tnode[cindex];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) /* this line carries forward the xor from earlier in the function */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) index = key ^ n->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) /* Step 3: Process the leaf, if that fails fall back to backtracing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) hlist_for_each_entry_rcu(fa, &n->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) struct fib_info *fi = fa->fa_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) struct fib_nh_common *nhc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) int nhsel, err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) if ((BITS_PER_LONG > KEYLENGTH) || (fa->fa_slen < KEYLENGTH)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) if (index >= (1ul << fa->fa_slen))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) if (fa->fa_tos && fa->fa_tos != flp->flowi4_tos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) if (fi->fib_dead)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) if (fa->fa_info->fib_scope < flp->flowi4_scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) fib_alias_accessed(fa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) err = fib_props[fa->fa_type].error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) if (unlikely(err < 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) out_reject:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) this_cpu_inc(stats->semantic_match_passed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) trace_fib_table_lookup(tb->tb_id, flp, NULL, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) if (fi->fib_flags & RTNH_F_DEAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) if (unlikely(fi->nh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) if (nexthop_is_blackhole(fi->nh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) err = fib_props[RTN_BLACKHOLE].error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) goto out_reject;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) nhc = nexthop_get_nhc_lookup(fi->nh, fib_flags, flp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) &nhsel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) if (nhc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) goto set_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) goto miss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) for (nhsel = 0; nhsel < fib_info_num_path(fi); nhsel++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) nhc = fib_info_nhc(fi, nhsel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) if (!fib_lookup_good_nhc(nhc, fib_flags, flp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) set_result:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) if (!(fib_flags & FIB_LOOKUP_NOREF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) refcount_inc(&fi->fib_clntref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) res->prefix = htonl(n->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) res->prefixlen = KEYLENGTH - fa->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) res->nh_sel = nhsel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) res->nhc = nhc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) res->type = fa->fa_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) res->scope = fi->fib_scope;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) res->fi = fi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) res->table = tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) res->fa_head = &n->leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) this_cpu_inc(stats->semantic_match_passed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) trace_fib_table_lookup(tb->tb_id, flp, nhc, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) miss:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) this_cpu_inc(stats->semantic_match_miss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) goto backtrace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) EXPORT_SYMBOL_GPL(fib_table_lookup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) static void fib_remove_alias(struct trie *t, struct key_vector *tp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) struct key_vector *l, struct fib_alias *old)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) /* record the location of the previous list_info entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) struct hlist_node **pprev = old->fa_list.pprev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) struct fib_alias *fa = hlist_entry(pprev, typeof(*fa), fa_list.next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) /* remove the fib_alias from the list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) hlist_del_rcu(&old->fa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) /* if we emptied the list this leaf will be freed and we can sort
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) * out parent suffix lengths as a part of trie_rebalance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) if (hlist_empty(&l->leaf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) if (tp->slen == l->slen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) node_pull_suffix(tp, tp->pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) put_child_root(tp, l->key, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) node_free(l);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) trie_rebalance(t, tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) /* only access fa if it is pointing at the last valid hlist_node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) if (*pprev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) /* update the trie with the latest suffix length */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) l->slen = fa->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) node_pull_suffix(tp, fa->fa_slen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) static void fib_notify_alias_delete(struct net *net, u32 key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) struct hlist_head *fah,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) struct fib_alias *fa_to_delete,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) struct fib_alias *fa_next, *fa_to_notify;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) u32 tb_id = fa_to_delete->tb_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) u8 slen = fa_to_delete->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) enum fib_event_type fib_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) /* Do not notify if we do not care about the route. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) if (fib_find_alias(fah, slen, 0, 0, tb_id, true) != fa_to_delete)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) /* Determine if the route should be replaced by the next route in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) * list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) fa_next = hlist_entry_safe(fa_to_delete->fa_list.next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) struct fib_alias, fa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) if (fa_next && fa_next->fa_slen == slen && fa_next->tb_id == tb_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) fib_event = FIB_EVENT_ENTRY_REPLACE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) fa_to_notify = fa_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) fib_event = FIB_EVENT_ENTRY_DEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) fa_to_notify = fa_to_delete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) call_fib_entry_notifiers(net, fib_event, key, KEYLENGTH - slen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) fa_to_notify, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) /* Caller must hold RTNL. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) int fib_table_delete(struct net *net, struct fib_table *tb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) struct fib_config *cfg, struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) struct trie *t = (struct trie *) tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) struct fib_alias *fa, *fa_to_delete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) struct key_vector *l, *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) u8 plen = cfg->fc_dst_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) u8 slen = KEYLENGTH - plen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) u8 tos = cfg->fc_tos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) u32 key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) key = ntohl(cfg->fc_dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) if (!fib_valid_key_len(key, plen, extack))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) l = fib_find_node(t, &tp, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) if (!l)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) return -ESRCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) fa = fib_find_alias(&l->leaf, slen, tos, 0, tb->tb_id, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) if (!fa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) return -ESRCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) pr_debug("Deleting %08x/%d tos=%d t=%p\n", key, plen, tos, t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) fa_to_delete = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) hlist_for_each_entry_from(fa, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) struct fib_info *fi = fa->fa_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) if ((fa->fa_slen != slen) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) (fa->tb_id != tb->tb_id) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) (fa->fa_tos != tos))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) if ((!cfg->fc_type || fa->fa_type == cfg->fc_type) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) (cfg->fc_scope == RT_SCOPE_NOWHERE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) fa->fa_info->fib_scope == cfg->fc_scope) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) (!cfg->fc_prefsrc ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) fi->fib_prefsrc == cfg->fc_prefsrc) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) (!cfg->fc_protocol ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) fi->fib_protocol == cfg->fc_protocol) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) fib_nh_match(net, cfg, fi, extack) == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) fib_metrics_match(cfg, fi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) fa_to_delete = fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) if (!fa_to_delete)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) return -ESRCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) fib_notify_alias_delete(net, key, &l->leaf, fa_to_delete, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) rtmsg_fib(RTM_DELROUTE, htonl(key), fa_to_delete, plen, tb->tb_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) &cfg->fc_nlinfo, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) if (!plen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) tb->tb_num_default--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) fib_remove_alias(t, tp, l, fa_to_delete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) if (fa_to_delete->fa_state & FA_S_ACCESSED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) rt_cache_flush(cfg->fc_nlinfo.nl_net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) fib_release_info(fa_to_delete->fa_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) alias_free_mem_rcu(fa_to_delete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) /* Scan for the next leaf starting at the provided key value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) static struct key_vector *leaf_walk_rcu(struct key_vector **tn, t_key key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) struct key_vector *pn, *n = *tn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) unsigned long cindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) /* this loop is meant to try and find the key in the trie */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) /* record parent and next child index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) pn = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) cindex = (key > pn->key) ? get_index(key, pn) : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) if (cindex >> pn->bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) /* descend into the next child */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) n = get_child_rcu(pn, cindex++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) /* guarantee forward progress on the keys */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) if (IS_LEAF(n) && (n->key >= key))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) } while (IS_TNODE(n));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) /* this loop will search for the next leaf with a greater key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) while (!IS_TRIE(pn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) /* if we exhausted the parent node we will need to climb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) if (cindex >= (1ul << pn->bits)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) t_key pkey = pn->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) pn = node_parent_rcu(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) cindex = get_index(pkey, pn) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) /* grab the next available node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) n = get_child_rcu(pn, cindex++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) /* no need to compare keys since we bumped the index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) if (IS_LEAF(n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) /* Rescan start scanning in new node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) pn = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) cindex = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) *tn = pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) return NULL; /* Root of trie */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) /* if we are at the limit for keys just return NULL for the tnode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) *tn = pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) static void fib_trie_free(struct fib_table *tb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) struct trie *t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) struct key_vector *pn = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) unsigned long cindex = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) struct hlist_node *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) /* walk trie in reverse order and free everything */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) struct key_vector *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) if (!(cindex--)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) t_key pkey = pn->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) if (IS_TRIE(pn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) n = pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) pn = node_parent(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) /* drop emptied tnode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) put_child_root(pn, n->key, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) node_free(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) cindex = get_index(pkey, pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) /* grab the next available node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) n = get_child(pn, cindex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) if (IS_TNODE(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) /* record pn and cindex for leaf walking */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) pn = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) cindex = 1ul << n->bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) hlist_for_each_entry_safe(fa, tmp, &n->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) hlist_del_rcu(&fa->fa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) alias_free_mem_rcu(fa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) put_child_root(pn, n->key, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) node_free(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) free_percpu(t->stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) kfree(tb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) struct fib_table *fib_trie_unmerge(struct fib_table *oldtb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) struct trie *ot = (struct trie *)oldtb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) struct key_vector *l, *tp = ot->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) struct fib_table *local_tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) struct trie *lt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) t_key key = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) if (oldtb->tb_data == oldtb->__data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) return oldtb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) local_tb = fib_trie_table(RT_TABLE_LOCAL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) if (!local_tb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) lt = (struct trie *)local_tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) while ((l = leaf_walk_rcu(&tp, key)) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) struct key_vector *local_l = NULL, *local_tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) hlist_for_each_entry(fa, &l->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) struct fib_alias *new_fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) if (local_tb->tb_id != fa->tb_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) /* clone fa for new local table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) new_fa = kmem_cache_alloc(fn_alias_kmem, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) if (!new_fa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) memcpy(new_fa, fa, sizeof(*fa));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) /* insert clone into table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) if (!local_l)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) local_l = fib_find_node(lt, &local_tp, l->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) if (fib_insert_alias(lt, local_tp, local_l, new_fa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) NULL, l->key)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) kmem_cache_free(fn_alias_kmem, new_fa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) /* stop loop if key wrapped back to 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) key = l->key + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) if (key < l->key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) return local_tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) fib_trie_free(local_tb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) /* Caller must hold RTNL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) void fib_table_flush_external(struct fib_table *tb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) struct trie *t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) struct key_vector *pn = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) unsigned long cindex = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) struct hlist_node *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) /* walk trie in reverse order */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) unsigned char slen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) struct key_vector *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) if (!(cindex--)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) t_key pkey = pn->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) /* cannot resize the trie vector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) if (IS_TRIE(pn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) /* update the suffix to address pulled leaves */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) if (pn->slen > pn->pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) update_suffix(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) /* resize completed node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) pn = resize(t, pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) cindex = get_index(pkey, pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) /* grab the next available node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) n = get_child(pn, cindex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) if (IS_TNODE(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) /* record pn and cindex for leaf walking */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) pn = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) cindex = 1ul << n->bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) hlist_for_each_entry_safe(fa, tmp, &n->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) /* if alias was cloned to local then we just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) * need to remove the local copy from main
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) if (tb->tb_id != fa->tb_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) hlist_del_rcu(&fa->fa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) alias_free_mem_rcu(fa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) /* record local slen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) slen = fa->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) /* update leaf slen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) n->slen = slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) if (hlist_empty(&n->leaf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) put_child_root(pn, n->key, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) node_free(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) /* Caller must hold RTNL. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) int fib_table_flush(struct net *net, struct fib_table *tb, bool flush_all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) struct trie *t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) struct key_vector *pn = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) unsigned long cindex = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) struct hlist_node *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) int found = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) /* walk trie in reverse order */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) unsigned char slen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) struct key_vector *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) if (!(cindex--)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) t_key pkey = pn->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) /* cannot resize the trie vector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) if (IS_TRIE(pn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) /* update the suffix to address pulled leaves */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) if (pn->slen > pn->pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) update_suffix(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) /* resize completed node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) pn = resize(t, pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) cindex = get_index(pkey, pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) /* grab the next available node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) n = get_child(pn, cindex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) if (IS_TNODE(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) /* record pn and cindex for leaf walking */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) pn = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) cindex = 1ul << n->bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) hlist_for_each_entry_safe(fa, tmp, &n->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) struct fib_info *fi = fa->fa_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) if (!fi || tb->tb_id != fa->tb_id ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) (!(fi->fib_flags & RTNH_F_DEAD) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) !fib_props[fa->fa_type].error)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) slen = fa->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) /* Do not flush error routes if network namespace is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) * not being dismantled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) if (!flush_all && fib_props[fa->fa_type].error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) slen = fa->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) fib_notify_alias_delete(net, n->key, &n->leaf, fa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) hlist_del_rcu(&fa->fa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) fib_release_info(fa->fa_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) alias_free_mem_rcu(fa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) found++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) /* update leaf slen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) n->slen = slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) if (hlist_empty(&n->leaf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) put_child_root(pn, n->key, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) node_free(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) pr_debug("trie_flush found=%d\n", found);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) /* derived from fib_trie_free */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) static void __fib_info_notify_update(struct net *net, struct fib_table *tb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) struct nl_info *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) struct trie *t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) struct key_vector *pn = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) unsigned long cindex = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) struct key_vector *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) if (!(cindex--)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) t_key pkey = pn->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) if (IS_TRIE(pn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) pn = node_parent(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) cindex = get_index(pkey, pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) /* grab the next available node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) n = get_child(pn, cindex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) if (IS_TNODE(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) /* record pn and cindex for leaf walking */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) pn = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) cindex = 1ul << n->bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) hlist_for_each_entry(fa, &n->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) struct fib_info *fi = fa->fa_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) if (!fi || !fi->nh_updated || fa->tb_id != tb->tb_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) rtmsg_fib(RTM_NEWROUTE, htonl(n->key), fa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) KEYLENGTH - fa->fa_slen, tb->tb_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) info, NLM_F_REPLACE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) /* call_fib_entry_notifiers will be removed when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) * in-kernel notifier is implemented and supported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) * for nexthop objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) call_fib_entry_notifiers(net, FIB_EVENT_ENTRY_REPLACE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) n->key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) KEYLENGTH - fa->fa_slen, fa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) void fib_info_notify_update(struct net *net, struct nl_info *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) unsigned int h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) struct hlist_head *head = &net->ipv4.fib_table_hash[h];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) struct fib_table *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) hlist_for_each_entry_rcu(tb, head, tb_hlist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) lockdep_rtnl_is_held())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) __fib_info_notify_update(net, tb, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) static int fib_leaf_notify(struct key_vector *l, struct fib_table *tb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) struct notifier_block *nb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) int last_slen = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) hlist_for_each_entry_rcu(fa, &l->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) struct fib_info *fi = fa->fa_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) if (!fi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) /* local and main table can share the same trie,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) * so don't notify twice for the same entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) if (tb->tb_id != fa->tb_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) if (fa->fa_slen == last_slen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) last_slen = fa->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) err = call_fib_entry_notifier(nb, FIB_EVENT_ENTRY_REPLACE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) l->key, KEYLENGTH - fa->fa_slen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) fa, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) static int fib_table_notify(struct fib_table *tb, struct notifier_block *nb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) struct trie *t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) struct key_vector *l, *tp = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) t_key key = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) while ((l = leaf_walk_rcu(&tp, key)) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) err = fib_leaf_notify(l, tb, nb, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) key = l->key + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) /* stop in case of wrap around */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) if (key < l->key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) int fib_notify(struct net *net, struct notifier_block *nb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) unsigned int h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) struct hlist_head *head = &net->ipv4.fib_table_hash[h];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) struct fib_table *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) hlist_for_each_entry_rcu(tb, head, tb_hlist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) err = fib_table_notify(tb, nb, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) static void __trie_free_rcu(struct rcu_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) struct fib_table *tb = container_of(head, struct fib_table, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) struct trie *t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) if (tb->tb_data == tb->__data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) free_percpu(t->stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) #endif /* CONFIG_IP_FIB_TRIE_STATS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) kfree(tb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) void fib_free_table(struct fib_table *tb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) call_rcu(&tb->rcu, __trie_free_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) static int fn_trie_dump_leaf(struct key_vector *l, struct fib_table *tb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) struct sk_buff *skb, struct netlink_callback *cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) struct fib_dump_filter *filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) unsigned int flags = NLM_F_MULTI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) __be32 xkey = htonl(l->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) int i, s_i, i_fa, s_fa, err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) if (filter->filter_set ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) !filter->dump_exceptions || !filter->dump_routes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) flags |= NLM_F_DUMP_FILTERED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) s_i = cb->args[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) s_fa = cb->args[5];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) /* rcu_read_lock is hold by caller */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) hlist_for_each_entry_rcu(fa, &l->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) struct fib_info *fi = fa->fa_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) if (i < s_i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) i_fa = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) if (tb->tb_id != fa->tb_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) if (filter->filter_set) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) if (filter->rt_type && fa->fa_type != filter->rt_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) if ((filter->protocol &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) fi->fib_protocol != filter->protocol))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) if (filter->dev &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) !fib_info_nh_uses_dev(fi, filter->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) if (filter->dump_routes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) if (!s_fa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) struct fib_rt_info fri;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) fri.fi = fi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) fri.tb_id = tb->tb_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) fri.dst = xkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) fri.dst_len = KEYLENGTH - fa->fa_slen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) fri.tos = fa->fa_tos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) fri.type = fa->fa_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) fri.offload = fa->offload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) fri.trap = fa->trap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) err = fib_dump_info(skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) NETLINK_CB(cb->skb).portid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) cb->nlh->nlmsg_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) RTM_NEWROUTE, &fri, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) goto stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) i_fa++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) if (filter->dump_exceptions) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) err = fib_dump_info_fnhe(skb, cb, tb->tb_id, fi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) &i_fa, s_fa, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) goto stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) cb->args[4] = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) return skb->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) stop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) cb->args[4] = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) cb->args[5] = i_fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) /* rcu_read_lock needs to be hold by caller from readside */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) int fib_table_dump(struct fib_table *tb, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) struct netlink_callback *cb, struct fib_dump_filter *filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) struct trie *t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) struct key_vector *l, *tp = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) /* Dump starting at last key.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) * Note: 0.0.0.0/0 (ie default) is first key.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) int count = cb->args[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) t_key key = cb->args[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) /* First time here, count and key are both always 0. Count > 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) * and key == 0 means the dump has wrapped around and we are done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) if (count && !key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) return skb->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) while ((l = leaf_walk_rcu(&tp, key)) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) err = fn_trie_dump_leaf(l, tb, skb, cb, filter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) cb->args[3] = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) cb->args[2] = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) ++count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) key = l->key + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) memset(&cb->args[4], 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) sizeof(cb->args) - 4*sizeof(cb->args[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) /* stop loop if key wrapped back to 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) if (key < l->key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) cb->args[3] = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) cb->args[2] = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) return skb->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) void __init fib_trie_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) fn_alias_kmem = kmem_cache_create("ip_fib_alias",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) sizeof(struct fib_alias),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) 0, SLAB_PANIC, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) trie_leaf_kmem = kmem_cache_create("ip_fib_trie",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) LEAF_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) 0, SLAB_PANIC, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) struct fib_table *fib_trie_table(u32 id, struct fib_table *alias)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) struct fib_table *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) struct trie *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) size_t sz = sizeof(*tb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) if (!alias)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) sz += sizeof(struct trie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) tb = kzalloc(sz, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) if (!tb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) tb->tb_id = id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) tb->tb_num_default = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) tb->tb_data = (alias ? alias->__data : tb->__data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) if (alias)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) return tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) t = (struct trie *) tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) t->kv[0].pos = KEYLENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) t->kv[0].slen = KEYLENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) t->stats = alloc_percpu(struct trie_use_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) if (!t->stats) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) kfree(tb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) tb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) return tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) #ifdef CONFIG_PROC_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) /* Depth first Trie walk iterator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) struct fib_trie_iter {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) struct seq_net_private p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) struct fib_table *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) struct key_vector *tnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) unsigned int index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) unsigned int depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) static struct key_vector *fib_trie_get_next(struct fib_trie_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) unsigned long cindex = iter->index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) struct key_vector *pn = iter->tnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) t_key pkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) pr_debug("get_next iter={node=%p index=%d depth=%d}\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) iter->tnode, iter->index, iter->depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) while (!IS_TRIE(pn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) while (cindex < child_length(pn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) struct key_vector *n = get_child_rcu(pn, cindex++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) if (IS_LEAF(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) iter->tnode = pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) iter->index = cindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) /* push down one level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) iter->tnode = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) iter->index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) ++iter->depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) /* Current node exhausted, pop back up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) pkey = pn->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) pn = node_parent_rcu(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) cindex = get_index(pkey, pn) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) --iter->depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) /* record root node so further searches know we are done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) iter->tnode = pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) iter->index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) static struct key_vector *fib_trie_get_first(struct fib_trie_iter *iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) struct trie *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) struct key_vector *n, *pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) if (!t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) pn = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) n = rcu_dereference(pn->tnode[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) if (IS_TNODE(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) iter->tnode = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) iter->index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) iter->depth = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) iter->tnode = pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) iter->index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) iter->depth = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) static void trie_collect_stats(struct trie *t, struct trie_stat *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) struct key_vector *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) struct fib_trie_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) memset(s, 0, sizeof(*s));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) for (n = fib_trie_get_first(&iter, t); n; n = fib_trie_get_next(&iter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) if (IS_LEAF(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) s->leaves++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) s->totdepth += iter.depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) if (iter.depth > s->maxdepth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) s->maxdepth = iter.depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) hlist_for_each_entry_rcu(fa, &n->leaf, fa_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) ++s->prefixes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) s->tnodes++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) if (n->bits < MAX_STAT_DEPTH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) s->nodesizes[n->bits]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) s->nullpointers += tn_info(n)->empty_children;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) * This outputs /proc/net/fib_triestats
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) static void trie_show_stats(struct seq_file *seq, struct trie_stat *stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) unsigned int i, max, pointers, bytes, avdepth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) if (stat->leaves)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) avdepth = stat->totdepth*100 / stat->leaves;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) avdepth = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) seq_printf(seq, "\tAver depth: %u.%02d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) avdepth / 100, avdepth % 100);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) seq_printf(seq, "\tMax depth: %u\n", stat->maxdepth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) seq_printf(seq, "\tLeaves: %u\n", stat->leaves);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) bytes = LEAF_SIZE * stat->leaves;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) seq_printf(seq, "\tPrefixes: %u\n", stat->prefixes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) bytes += sizeof(struct fib_alias) * stat->prefixes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) seq_printf(seq, "\tInternal nodes: %u\n\t", stat->tnodes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) bytes += TNODE_SIZE(0) * stat->tnodes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) max = MAX_STAT_DEPTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) while (max > 0 && stat->nodesizes[max-1] == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) max--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) pointers = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) for (i = 1; i < max; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) if (stat->nodesizes[i] != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) seq_printf(seq, " %u: %u", i, stat->nodesizes[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) pointers += (1<<i) * stat->nodesizes[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) seq_putc(seq, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) seq_printf(seq, "\tPointers: %u\n", pointers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) bytes += sizeof(struct key_vector *) * pointers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) seq_printf(seq, "Null ptrs: %u\n", stat->nullpointers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) seq_printf(seq, "Total size: %u kB\n", (bytes + 1023) / 1024);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) static void trie_show_usage(struct seq_file *seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) const struct trie_use_stats __percpu *stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) struct trie_use_stats s = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) /* loop through all of the CPUs and gather up the stats */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) const struct trie_use_stats *pcpu = per_cpu_ptr(stats, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) s.gets += pcpu->gets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) s.backtrack += pcpu->backtrack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) s.semantic_match_passed += pcpu->semantic_match_passed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) s.semantic_match_miss += pcpu->semantic_match_miss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) s.null_node_hit += pcpu->null_node_hit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) s.resize_node_skipped += pcpu->resize_node_skipped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) seq_printf(seq, "\nCounters:\n---------\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) seq_printf(seq, "gets = %u\n", s.gets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) seq_printf(seq, "backtracks = %u\n", s.backtrack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) seq_printf(seq, "semantic match passed = %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) s.semantic_match_passed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) seq_printf(seq, "semantic match miss = %u\n", s.semantic_match_miss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) seq_printf(seq, "null node hit= %u\n", s.null_node_hit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) seq_printf(seq, "skipped node resize = %u\n\n", s.resize_node_skipped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) #endif /* CONFIG_IP_FIB_TRIE_STATS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) static void fib_table_print(struct seq_file *seq, struct fib_table *tb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) if (tb->tb_id == RT_TABLE_LOCAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) seq_puts(seq, "Local:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) else if (tb->tb_id == RT_TABLE_MAIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) seq_puts(seq, "Main:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) seq_printf(seq, "Id %d:\n", tb->tb_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) static int fib_triestat_seq_show(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) struct net *net = (struct net *)seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) unsigned int h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) seq_printf(seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) "Basic info: size of leaf:"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) " %zd bytes, size of tnode: %zd bytes.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) LEAF_SIZE, TNODE_SIZE(0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) struct hlist_head *head = &net->ipv4.fib_table_hash[h];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) struct fib_table *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) hlist_for_each_entry_rcu(tb, head, tb_hlist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) struct trie *t = (struct trie *) tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) struct trie_stat stat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) if (!t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) fib_table_print(seq, tb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) trie_collect_stats(t, &stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) trie_show_stats(seq, &stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) #ifdef CONFIG_IP_FIB_TRIE_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) trie_show_usage(seq, t->stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) cond_resched_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) static struct key_vector *fib_trie_get_idx(struct seq_file *seq, loff_t pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) struct fib_trie_iter *iter = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) struct net *net = seq_file_net(seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) loff_t idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) unsigned int h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) struct hlist_head *head = &net->ipv4.fib_table_hash[h];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) struct fib_table *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) hlist_for_each_entry_rcu(tb, head, tb_hlist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) struct key_vector *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) for (n = fib_trie_get_first(iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) (struct trie *) tb->tb_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) n; n = fib_trie_get_next(iter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) if (pos == idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) iter->tb = tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) static void *fib_trie_seq_start(struct seq_file *seq, loff_t *pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) __acquires(RCU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) return fib_trie_get_idx(seq, *pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) static void *fib_trie_seq_next(struct seq_file *seq, void *v, loff_t *pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) struct fib_trie_iter *iter = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) struct net *net = seq_file_net(seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) struct fib_table *tb = iter->tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) struct hlist_node *tb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) unsigned int h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) struct key_vector *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) ++*pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) /* next node in same table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) n = fib_trie_get_next(iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) if (n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) /* walk rest of this hash chain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) h = tb->tb_id & (FIB_TABLE_HASHSZ - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) while ((tb_node = rcu_dereference(hlist_next_rcu(&tb->tb_hlist)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) tb = hlist_entry(tb_node, struct fib_table, tb_hlist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) n = fib_trie_get_first(iter, (struct trie *) tb->tb_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) if (n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) /* new hash chain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) while (++h < FIB_TABLE_HASHSZ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) struct hlist_head *head = &net->ipv4.fib_table_hash[h];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) hlist_for_each_entry_rcu(tb, head, tb_hlist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) n = fib_trie_get_first(iter, (struct trie *) tb->tb_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) if (n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) iter->tb = tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) static void fib_trie_seq_stop(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) __releases(RCU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) static void seq_indent(struct seq_file *seq, int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) while (n-- > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) seq_puts(seq, " ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) static inline const char *rtn_scope(char *buf, size_t len, enum rt_scope_t s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) switch (s) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) case RT_SCOPE_UNIVERSE: return "universe";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) case RT_SCOPE_SITE: return "site";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) case RT_SCOPE_LINK: return "link";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) case RT_SCOPE_HOST: return "host";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) case RT_SCOPE_NOWHERE: return "nowhere";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) snprintf(buf, len, "scope=%d", s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) return buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) static const char *const rtn_type_names[__RTN_MAX] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) [RTN_UNSPEC] = "UNSPEC",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) [RTN_UNICAST] = "UNICAST",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) [RTN_LOCAL] = "LOCAL",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) [RTN_BROADCAST] = "BROADCAST",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) [RTN_ANYCAST] = "ANYCAST",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) [RTN_MULTICAST] = "MULTICAST",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) [RTN_BLACKHOLE] = "BLACKHOLE",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) [RTN_UNREACHABLE] = "UNREACHABLE",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) [RTN_PROHIBIT] = "PROHIBIT",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) [RTN_THROW] = "THROW",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) [RTN_NAT] = "NAT",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) [RTN_XRESOLVE] = "XRESOLVE",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) static inline const char *rtn_type(char *buf, size_t len, unsigned int t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) if (t < __RTN_MAX && rtn_type_names[t])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) return rtn_type_names[t];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) snprintf(buf, len, "type %u", t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) return buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) /* Pretty print the trie */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) static int fib_trie_seq_show(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) const struct fib_trie_iter *iter = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) struct key_vector *n = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) if (IS_TRIE(node_parent_rcu(n)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) fib_table_print(seq, iter->tb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) if (IS_TNODE(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) __be32 prf = htonl(n->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) seq_indent(seq, iter->depth-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) seq_printf(seq, " +-- %pI4/%zu %u %u %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) &prf, KEYLENGTH - n->pos - n->bits, n->bits,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) tn_info(n)->full_children,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) tn_info(n)->empty_children);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) __be32 val = htonl(n->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) seq_indent(seq, iter->depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) seq_printf(seq, " |-- %pI4\n", &val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) hlist_for_each_entry_rcu(fa, &n->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) char buf1[32], buf2[32];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) seq_indent(seq, iter->depth + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) seq_printf(seq, " /%zu %s %s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) KEYLENGTH - fa->fa_slen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) rtn_scope(buf1, sizeof(buf1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) fa->fa_info->fib_scope),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) rtn_type(buf2, sizeof(buf2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) fa->fa_type));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) if (fa->fa_tos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) seq_printf(seq, " tos=%d", fa->fa_tos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) seq_putc(seq, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) static const struct seq_operations fib_trie_seq_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) .start = fib_trie_seq_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) .next = fib_trie_seq_next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) .stop = fib_trie_seq_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) .show = fib_trie_seq_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) struct fib_route_iter {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) struct seq_net_private p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) struct fib_table *main_tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) struct key_vector *tnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) loff_t pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) t_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) loff_t pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) struct key_vector *l, **tp = &iter->tnode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) t_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) /* use cached location of previously found key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) if (iter->pos > 0 && pos >= iter->pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) key = iter->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) iter->pos = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) key = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) pos -= iter->pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) while ((l = leaf_walk_rcu(tp, key)) && (pos-- > 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) key = l->key + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) iter->pos++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) l = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) /* handle unlikely case of a key wrap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) if (!key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) if (l)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) iter->key = l->key; /* remember it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) iter->pos = 0; /* forget it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) return l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) static void *fib_route_seq_start(struct seq_file *seq, loff_t *pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) __acquires(RCU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) struct fib_route_iter *iter = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) struct fib_table *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) struct trie *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) tb = fib_get_table(seq_file_net(seq), RT_TABLE_MAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) if (!tb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) iter->main_tb = tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) t = (struct trie *)tb->tb_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) iter->tnode = t->kv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) if (*pos != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) return fib_route_get_idx(iter, *pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) iter->pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) iter->key = KEY_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) return SEQ_START_TOKEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) struct fib_route_iter *iter = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) struct key_vector *l = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) t_key key = iter->key + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) ++*pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) /* only allow key of 0 for start of sequence */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) if ((v == SEQ_START_TOKEN) || key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) l = leaf_walk_rcu(&iter->tnode, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) if (l) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) iter->key = l->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) iter->pos++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) iter->pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) return l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) static void fib_route_seq_stop(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) __releases(RCU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) static unsigned int fib_flag_trans(int type, __be32 mask, struct fib_info *fi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) unsigned int flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) if (type == RTN_UNREACHABLE || type == RTN_PROHIBIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) flags = RTF_REJECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) if (fi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) if (nhc->nhc_gw.ipv4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) flags |= RTF_GATEWAY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) if (mask == htonl(0xFFFFFFFF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) flags |= RTF_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) flags |= RTF_UP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) return flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) * This outputs /proc/net/route.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) * The format of the file is not supposed to be changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) * and needs to be same as fib_hash output to avoid breaking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) * legacy utilities
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) static int fib_route_seq_show(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) struct fib_route_iter *iter = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) struct fib_table *tb = iter->main_tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) struct fib_alias *fa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) struct key_vector *l = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) __be32 prefix;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) if (v == SEQ_START_TOKEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) seq_printf(seq, "%-127s\n", "Iface\tDestination\tGateway "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) "\tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) "\tWindow\tIRTT");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) prefix = htonl(l->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) hlist_for_each_entry_rcu(fa, &l->leaf, fa_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) struct fib_info *fi = fa->fa_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) __be32 mask = inet_make_mask(KEYLENGTH - fa->fa_slen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) unsigned int flags = fib_flag_trans(fa->fa_type, mask, fi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) if ((fa->fa_type == RTN_BROADCAST) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) (fa->fa_type == RTN_MULTICAST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) if (fa->tb_id != tb->tb_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) seq_setwidth(seq, 127);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) if (fi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) struct fib_nh_common *nhc = fib_info_nhc(fi, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) __be32 gw = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) if (nhc->nhc_gw_family == AF_INET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) gw = nhc->nhc_gw.ipv4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) seq_printf(seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) "%s\t%08X\t%08X\t%04X\t%d\t%u\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) "%d\t%08X\t%d\t%u\t%u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) nhc->nhc_dev ? nhc->nhc_dev->name : "*",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) prefix, gw, flags, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) fi->fib_priority,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) (fi->fib_advmss ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) fi->fib_advmss + 40 : 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) fi->fib_window,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) fi->fib_rtt >> 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) seq_printf(seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) "*\t%08X\t%08X\t%04X\t%d\t%u\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) "%d\t%08X\t%d\t%u\t%u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) prefix, 0, flags, 0, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) mask, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) seq_pad(seq, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) static const struct seq_operations fib_route_seq_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) .start = fib_route_seq_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) .next = fib_route_seq_next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) .stop = fib_route_seq_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) .show = fib_route_seq_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) int __net_init fib_proc_init(struct net *net)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) if (!proc_create_net("fib_trie", 0444, net->proc_net, &fib_trie_seq_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) sizeof(struct fib_trie_iter)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) goto out1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) if (!proc_create_net_single("fib_triestat", 0444, net->proc_net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) fib_triestat_seq_show, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) goto out2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) if (!proc_create_net("route", 0444, net->proc_net, &fib_route_seq_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) sizeof(struct fib_route_iter)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) goto out3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) out3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) remove_proc_entry("fib_triestat", net->proc_net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) out2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) remove_proc_entry("fib_trie", net->proc_net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) out1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) void __net_exit fib_proc_exit(struct net *net)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) remove_proc_entry("fib_trie", net->proc_net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) remove_proc_entry("fib_triestat", net->proc_net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) remove_proc_entry("route", net->proc_net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) #endif /* CONFIG_PROC_FS */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags