Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * INET		An implementation of the TCP/IP protocol suite for the LINUX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *		operating system.  INET is implemented using the BSD Socket
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *		interface as the means of communication with the user level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *		Generic INET6 transport hashtables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * Authors:	Lotsa people, from code originally in tcp, generalised here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  *		by Arnaldo Carvalho de Melo <acme@mandriva.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/random.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <net/addrconf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <net/inet_connection_sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <net/inet_hashtables.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <net/inet6_hashtables.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <net/secure_seq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <net/ip.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <net/sock_reuseport.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) extern struct inet_hashinfo tcp_hashinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) u32 inet6_ehashfn(const struct net *net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 		  const struct in6_addr *laddr, const u16 lport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 		  const struct in6_addr *faddr, const __be16 fport)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	static u32 inet6_ehash_secret __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	static u32 ipv6_hash_secret __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	u32 lhash, fhash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	lhash = (__force u32)laddr->s6_addr32[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	return __inet6_ehashfn(lhash, lport, fhash, fport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 			       inet6_ehash_secret + net_hash_mix(net));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49)  * The sockhash lock must be held as a reader here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) struct sock *__inet6_lookup_established(struct net *net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 					struct inet_hashinfo *hashinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 					   const struct in6_addr *saddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 					   const __be16 sport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 					   const struct in6_addr *daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 					   const u16 hnum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 					   const int dif, const int sdif)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	struct sock *sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	const struct hlist_nulls_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	/* Optimize here for direct hit, only listening connections can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	 * have wildcards anyways.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	unsigned int slot = hash & hashinfo->ehash_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) begin:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	sk_nulls_for_each_rcu(sk, node, &head->chain) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 		if (sk->sk_hash != hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 		if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 		if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 			goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 		if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 			sock_gen_put(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 			goto begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 		goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	if (get_nulls_value(node) != slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		goto begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	sk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	return sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) EXPORT_SYMBOL(__inet6_lookup_established);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) static inline int compute_score(struct sock *sk, struct net *net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 				const unsigned short hnum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 				const struct in6_addr *daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 				const int dif, const int sdif)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	int score = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	    sk->sk_family == PF_INET6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 		if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 			return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 			return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 		score =  sk->sk_bound_dev_if ? 2 : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 		if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 			score++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	return score;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static inline struct sock *lookup_reuseport(struct net *net, struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 					    struct sk_buff *skb, int doff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 					    const struct in6_addr *saddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 					    __be16 sport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 					    const struct in6_addr *daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 					    unsigned short hnum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	struct sock *reuse_sk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	u32 phash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	if (sk->sk_reuseport) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 		phash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		reuse_sk = reuseport_select_sock(sk, phash, skb, doff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	return reuse_sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /* called with rcu_read_lock() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) static struct sock *inet6_lhash2_lookup(struct net *net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 		struct inet_listen_hashbucket *ilb2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 		struct sk_buff *skb, int doff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 		const struct in6_addr *saddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 		const __be16 sport, const struct in6_addr *daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 		const unsigned short hnum, const int dif, const int sdif)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	struct inet_connection_sock *icsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	struct sock *sk, *result = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	int score, hiscore = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	inet_lhash2_for_each_icsk_rcu(icsk, &ilb2->head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 		sk = (struct sock *)icsk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 		score = compute_score(sk, net, hnum, daddr, dif, sdif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 		if (score > hiscore) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 			result = lookup_reuseport(net, sk, skb, doff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 						  saddr, sport, daddr, hnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 			if (result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 				return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 			result = sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 			hiscore = score;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static inline struct sock *inet6_lookup_run_bpf(struct net *net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 						struct inet_hashinfo *hashinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 						struct sk_buff *skb, int doff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 						const struct in6_addr *saddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 						const __be16 sport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 						const struct in6_addr *daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 						const u16 hnum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	struct sock *sk, *reuse_sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	bool no_reuseport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	if (hashinfo != &tcp_hashinfo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 		return NULL; /* only TCP is supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 					    saddr, sport, daddr, hnum, &sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	if (no_reuseport || IS_ERR_OR_NULL(sk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 		return sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	reuse_sk = lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	if (reuse_sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 		sk = reuse_sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	return sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct sock *inet6_lookup_listener(struct net *net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		struct inet_hashinfo *hashinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 		struct sk_buff *skb, int doff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 		const struct in6_addr *saddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		const __be16 sport, const struct in6_addr *daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 		const unsigned short hnum, const int dif, const int sdif)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	struct inet_listen_hashbucket *ilb2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	struct sock *result = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	unsigned int hash2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	/* Lookup redirect from BPF */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 		result = inet6_lookup_run_bpf(net, hashinfo, skb, doff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 					      saddr, sport, daddr, hnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 		if (result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 			goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	hash2 = ipv6_portaddr_hash(net, daddr, hnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	ilb2 = inet_lhash2_bucket(hashinfo, hash2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	result = inet6_lhash2_lookup(net, ilb2, skb, doff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 				     saddr, sport, daddr, hnum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 				     dif, sdif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	if (result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 		goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	/* Lookup lhash2 with in6addr_any */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	ilb2 = inet_lhash2_bucket(hashinfo, hash2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	result = inet6_lhash2_lookup(net, ilb2, skb, doff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 				     saddr, sport, &in6addr_any, hnum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 				     dif, sdif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	if (IS_ERR(result))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) EXPORT_SYMBOL_GPL(inet6_lookup_listener);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 			  struct sk_buff *skb, int doff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 			  const struct in6_addr *saddr, const __be16 sport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 			  const struct in6_addr *daddr, const __be16 dport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 			  const int dif)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	struct sock *sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	bool refcounted;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 			    ntohs(dport), dif, 0, &refcounted);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	if (sk && !refcounted && !refcount_inc_not_zero(&sk->sk_refcnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		sk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	return sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) EXPORT_SYMBOL_GPL(inet6_lookup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) static int __inet6_check_established(struct inet_timewait_death_row *death_row,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 				     struct sock *sk, const __u16 lport,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 				     struct inet_timewait_sock **twp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	struct inet_hashinfo *hinfo = death_row->hashinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	struct inet_sock *inet = inet_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	const struct in6_addr *saddr = &sk->sk_v6_daddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	const int dif = sk->sk_bound_dev_if;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	struct net *net = sock_net(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	const int sdif = l3mdev_master_ifindex_by_index(net, dif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 						inet->inet_dport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	struct sock *sk2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	const struct hlist_nulls_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	struct inet_timewait_sock *tw = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	spin_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	sk_nulls_for_each(sk2, node, &head->chain) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 		if (sk2->sk_hash != hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 		if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 				       dif, sdif))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 			if (sk2->sk_state == TCP_TIME_WAIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 				tw = inet_twsk(sk2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 				if (twsk_unique(sk, sk2, twp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 					break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 			goto not_unique;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	/* Must record num and sport now. Otherwise we will see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	 * in hash table socket with a funny identity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	inet->inet_num = lport;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	inet->inet_sport = htons(lport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	sk->sk_hash = hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	WARN_ON(!sk_unhashed(sk));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	__sk_nulls_add_node_rcu(sk, &head->chain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	if (tw) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 		sk_nulls_del_node_init_rcu((struct sock *)tw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 		__NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	spin_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	if (twp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 		*twp = tw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	} else if (tw) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 		/* Silly. Should hash-dance instead... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		inet_twsk_deschedule_put(tw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) not_unique:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	spin_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	return -EADDRNOTAVAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) static u32 inet6_sk_port_offset(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	const struct inet_sock *inet = inet_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 					  sk->sk_v6_daddr.s6_addr32,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 					  inet->inet_dport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) int inet6_hash_connect(struct inet_timewait_death_row *death_row,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 		       struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	u32 port_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	if (!inet_sk(sk)->inet_num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 		port_offset = inet6_sk_port_offset(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	return __inet_hash_connect(death_row, sk, port_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 				   __inet6_check_established);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) EXPORT_SYMBOL_GPL(inet6_hash_connect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) int inet6_hash(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	if (sk->sk_state != TCP_CLOSE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 		local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 		err = __inet_hash(sk, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 		local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 	return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) EXPORT_SYMBOL_GPL(inet6_hash);