^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-or-later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * INET An implementation of the TCP/IP protocol suite for the LINUX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * operating system. INET is implemented using the BSD Socket
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * interface as the means of communication with the user level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Definitions for the TCP module.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Version: @(#)tcp.h 1.0.5 05/23/93
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Authors: Ross Biro
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #ifndef _TCP_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #define _TCP_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define FASTRETRANS_DEBUG 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/tcp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/cache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/skbuff.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/kref.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/ktime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/indirect_call_wrapper.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <net/inet_connection_sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <net/inet_timewait_sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <net/inet_hashtables.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <net/checksum.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <net/request_sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <net/sock_reuseport.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <net/sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <net/snmp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <net/ip.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <net/tcp_states.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <net/inet_ecn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <net/dst.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <net/mptcp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/memcontrol.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/bpf-cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/siphash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) extern struct inet_hashinfo tcp_hashinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) extern struct percpu_counter tcp_orphan_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) void tcp_time_wait(struct sock *sk, int state, int timeo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define MAX_TCP_OPTION_SPACE 40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define TCP_MIN_SND_MSS 48
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * Never offer a window over 32767 without using window scaling. Some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * poor stacks do signed 16bit maths!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define MAX_TCP_WINDOW 32767U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /* Minimal accepted MSS. It is (60+60+8) - (20+20). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define TCP_MIN_MSS 88U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* The initial MTU to use for probing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define TCP_BASE_MSS 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /* probing interval, default to 10 minutes as per RFC4821 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define TCP_PROBE_INTERVAL 600
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) /* Specify interval when tcp mtu probing will stop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define TCP_PROBE_THRESHOLD 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /* After receiving this amount of duplicate ACKs fast retransmit starts. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define TCP_FASTRETRANS_THRESH 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /* Maximal number of ACKs sent quickly to accelerate slow-start. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #define TCP_MAX_QUICKACKS 16U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* Maximal number of window scale according to RFC1323 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define TCP_MAX_WSCALE 14U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) /* urg_data states */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define TCP_URG_VALID 0x0100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #define TCP_URG_NOTYET 0x0200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #define TCP_URG_READ 0x0400
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define TCP_RETR1 3 /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * This is how many retries it does before it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * tries to figure out if the gateway is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * down. Minimal RFC value is 3; it corresponds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * to ~3sec-8min depending on RTO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #define TCP_RETR2 15 /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * This should take at least
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * 90 minutes to time out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * RFC1122 says that the limit is 100 sec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * 15 is ~13-30min depending on RTO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define TCP_SYN_RETRIES 6 /* This is how many retries are done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * when active opening a connection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * RFC1122 says the minimum retry MUST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * be at least 180secs. Nevertheless
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * this value is corresponding to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * 63secs of retransmission with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * current initial RTO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #define TCP_SYNACK_RETRIES 5 /* This is how may retries are done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * when passive opening a connection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * This is corresponding to 31secs of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * retransmission with the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * initial RTO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * state, about 60 seconds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /* BSD style FIN_WAIT2 deadlock breaker.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * It used to be 3min, new value is 60sec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * to combine FIN-WAIT-2 timeout with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * TIME-WAIT timer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #if HZ >= 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define TCP_ATO_MIN ((unsigned)(HZ/25))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #define TCP_DELACK_MIN 4U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #define TCP_ATO_MIN 4U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #define TCP_RTO_MAX ((unsigned)(120*HZ))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #define TCP_RTO_MIN ((unsigned)(HZ/5))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * used as a fallback RTO for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * initial data transmission if no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * valid RTT sample has been acquired,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * most likely due to retrans in 3WHS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * for local resources.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #define TCP_KEEPALIVE_INTVL (75*HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) #define MAX_TCP_KEEPIDLE 32767
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #define MAX_TCP_KEEPINTVL 32767
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #define MAX_TCP_KEEPCNT 127
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #define MAX_TCP_SYNCNT 127
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #define TCP_SYNQ_INTERVAL (HZ/5) /* Period of SYNACK timer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #define TCP_PAWS_24DAYS (60 * 60 * 24 * 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * after this time. It should be equal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * (or greater than) TCP_TIMEWAIT_LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * to provide reliability equal to one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * provided by timewait state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) #define TCP_PAWS_WINDOW 1 /* Replay window for per-host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * timestamps. It must be less than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * minimal timewait lifetime.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * TCP option
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) #define TCPOPT_NOP 1 /* Padding */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) #define TCPOPT_EOL 0 /* End of options */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) #define TCPOPT_MSS 2 /* Segment size negotiating */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) #define TCPOPT_WINDOW 3 /* Window scaling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #define TCPOPT_SACK_PERM 4 /* SACK Permitted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #define TCPOPT_SACK 5 /* SACK Block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) #define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) #define TCPOPT_MPTCP 30 /* Multipath TCP (RFC6824) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) #define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) #define TCPOPT_EXP 254 /* Experimental */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /* Magic number to be after the option value for sharing TCP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * experimental options. See draft-ietf-tcpm-experimental-options-00.txt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #define TCPOPT_FASTOPEN_MAGIC 0xF989
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #define TCPOPT_SMC_MAGIC 0xE2D4C3D9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * TCP option lengths
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) #define TCPOLEN_MSS 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) #define TCPOLEN_WINDOW 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) #define TCPOLEN_SACK_PERM 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #define TCPOLEN_TIMESTAMP 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) #define TCPOLEN_MD5SIG 18
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) #define TCPOLEN_FASTOPEN_BASE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) #define TCPOLEN_EXP_FASTOPEN_BASE 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) #define TCPOLEN_EXP_SMC_BASE 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) /* But this is what stacks really send out. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #define TCPOLEN_TSTAMP_ALIGNED 12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) #define TCPOLEN_WSCALE_ALIGNED 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) #define TCPOLEN_SACKPERM_ALIGNED 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #define TCPOLEN_SACK_BASE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) #define TCPOLEN_SACK_BASE_ALIGNED 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) #define TCPOLEN_SACK_PERBLOCK 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) #define TCPOLEN_MD5SIG_ALIGNED 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) #define TCPOLEN_MSS_ALIGNED 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) #define TCPOLEN_EXP_SMC_BASE_ALIGNED 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /* Flags in tp->nonagle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) #define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) #define TCP_NAGLE_CORK 2 /* Socket is corked */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) #define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /* TCP thin-stream limits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) /* TCP initial congestion window as per rfc6928 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #define TCP_INIT_CWND 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) /* Bit Flags for sysctl_tcp_fastopen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) #define TFO_CLIENT_ENABLE 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) #define TFO_SERVER_ENABLE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) #define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /* Accept SYN data w/o any cookie option */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) #define TFO_SERVER_COOKIE_NOT_REQD 0x200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /* Force enable TFO on all listeners, i.e., not requiring the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * TCP_FASTOPEN socket option.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) #define TFO_SERVER_WO_SOCKOPT1 0x400
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /* sysctl variables for tcp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) extern int sysctl_tcp_max_orphans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) extern long sysctl_tcp_mem[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) #define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) extern atomic_long_t tcp_memory_allocated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) extern struct percpu_counter tcp_sockets_allocated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) extern unsigned long tcp_memory_pressure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /* optimized version of sk_under_memory_pressure() for TCP sockets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) static inline bool tcp_under_memory_pressure(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) mem_cgroup_under_socket_pressure(sk->sk_memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) return READ_ONCE(tcp_memory_pressure);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * The next routines deal with comparing 32 bit unsigned ints
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * and worry about wraparound (automatic with unsigned arithmetic).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static inline bool before(__u32 seq1, __u32 seq2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return (__s32)(seq1-seq2) < 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) #define after(seq2, seq1) before(seq1, seq2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /* is s2<=s1<=s3 ? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return seq3 - seq2 >= seq1 - seq2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) static inline bool tcp_out_of_memory(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) void sk_forced_mem_schedule(struct sock *sk, int size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static inline bool tcp_too_many_orphans(struct sock *sk, int shift)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct percpu_counter *ocp = sk->sk_prot->orphan_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) int orphans = percpu_counter_read_positive(ocp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (orphans << shift > sysctl_tcp_max_orphans) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) orphans = percpu_counter_sum_positive(ocp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (orphans << shift > sysctl_tcp_max_orphans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) bool tcp_check_oom(struct sock *sk, int shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) extern struct proto tcp_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) void tcp_tasklet_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) int tcp_v4_err(struct sk_buff *skb, u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) void tcp_shutdown(struct sock *sk, int how);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) int tcp_v4_early_demux(struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) int tcp_v4_rcv(struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) int flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) size_t size, int flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) size_t size, int flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) int size_goal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) void tcp_release_cb(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) void tcp_wfree(struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) void tcp_write_timer_handler(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) void tcp_delack_timer_handler(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) void tcp_rcv_space_adjust(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) void tcp_twsk_destructor(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) struct pipe_inode_info *pipe, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) unsigned int flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static inline void tcp_dec_quickack_mode(struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) const unsigned int pkts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) struct inet_connection_sock *icsk = inet_csk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (icsk->icsk_ack.quick) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (pkts >= icsk->icsk_ack.quick) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) icsk->icsk_ack.quick = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) /* Leaving quickack mode we deflate ATO. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) icsk->icsk_ack.ato = TCP_ATO_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) icsk->icsk_ack.quick -= pkts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) #define TCP_ECN_OK 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) #define TCP_ECN_QUEUE_CWR 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) #define TCP_ECN_DEMAND_CWR 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) #define TCP_ECN_SEEN 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) enum tcp_tw_status {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) TCP_TW_SUCCESS = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) TCP_TW_RST = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) TCP_TW_ACK = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) TCP_TW_SYN = 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) const struct tcphdr *th);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) struct request_sock *req, bool fastopen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) bool *lost_race);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) int tcp_child_process(struct sock *parent, struct sock *child,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) void tcp_enter_loss(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) void tcp_clear_retrans(struct tcp_sock *tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) void tcp_update_metrics(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) void tcp_init_metrics(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) void tcp_metrics_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) void tcp_close(struct sock *sk, long timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) void tcp_init_sock(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) __poll_t tcp_poll(struct file *file, struct socket *sock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) struct poll_table_struct *wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) int tcp_getsockopt(struct sock *sk, int level, int optname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) char __user *optval, int __user *optlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) unsigned int optlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) void tcp_set_keepalive(struct sock *sk, int val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) void tcp_syn_ack_timeout(const struct request_sock *req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) int flags, int *addr_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) int tcp_set_rcvlowat(struct sock *sk, int val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) void tcp_data_ready(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) #ifdef CONFIG_MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) int tcp_mmap(struct file *file, struct socket *sock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) struct vm_area_struct *vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct tcp_options_received *opt_rx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) int estab, struct tcp_fastopen_cookie *foc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) const u8 *tcp_parse_md5sig_option(const struct tcphdr *th);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * BPF SKB-less helpers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) struct tcphdr *th, u32 *cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) struct tcphdr *th, u32 *cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) const struct tcp_request_sock_ops *af_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct sock *sk, struct tcphdr *th);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * TCP v4 functions exported for the inet6 API
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) void tcp_v4_mtu_reduced(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) void tcp_req_err(struct sock *sk, u32 seq, bool abort);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) void tcp_ld_RTO_revert(struct sock *sk, u32 seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) struct sock *tcp_create_openreq_child(const struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) struct request_sock *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) struct request_sock *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) struct dst_entry *dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) struct request_sock *req_unhash,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) bool *own_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) int tcp_connect(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) enum tcp_synack_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) TCP_SYNACK_NORMAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) TCP_SYNACK_FASTOPEN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) TCP_SYNACK_COOKIE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) struct request_sock *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) struct tcp_fastopen_cookie *foc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) enum tcp_synack_type synack_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) struct sk_buff *syn_skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) int tcp_disconnect(struct sock *sk, int flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) /* From syncookies.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct request_sock *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) struct dst_entry *dst, u32 tsoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) u32 cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) #ifdef CONFIG_SYN_COOKIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) /* Syncookies use a monotonic timer which increments every 60 seconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * This counter is used both as a hash input and partially encoded into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * the cookie value. A cookie is only validated further if the delta
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * between the current counter value and the encoded one is less than this,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * the counter advances immediately after a cookie is generated).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) #define MAX_SYNCOOKIE_AGE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) #define TCP_SYNCOOKIE_PERIOD (60 * HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /* syncookies: remember time of last synqueue overflow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * But do not dirty this field too often (once per second is enough)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * It is racy as we do not hold a lock, but race is very minor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) static inline void tcp_synq_overflow(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) unsigned int last_overflow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) unsigned int now = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (sk->sk_reuseport) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) struct sock_reuseport *reuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) reuse = rcu_dereference(sk->sk_reuseport_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (likely(reuse)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) last_overflow = READ_ONCE(reuse->synq_overflow_ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (!time_between32(now, last_overflow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) last_overflow + HZ))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) WRITE_ONCE(reuse->synq_overflow_ts, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) if (!time_between32(now, last_overflow, last_overflow + HZ))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) WRITE_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) /* syncookies: no recent synqueue overflow on this listening socket? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) unsigned int last_overflow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) unsigned int now = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (sk->sk_reuseport) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) struct sock_reuseport *reuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) reuse = rcu_dereference(sk->sk_reuseport_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) if (likely(reuse)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) last_overflow = READ_ONCE(reuse->synq_overflow_ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return !time_between32(now, last_overflow - HZ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) last_overflow +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) TCP_SYNCOOKIE_VALID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) /* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) * then we're under synflood. However, we have to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * 'last_overflow - HZ' as lower bound. That's because a concurrent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * tcp_synq_overflow() could update .ts_recent_stamp after we read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * jiffies but before we store .ts_recent_stamp into last_overflow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * which could lead to rejecting a valid syncookie.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) return !time_between32(now, last_overflow - HZ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) last_overflow + TCP_SYNCOOKIE_VALID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) static inline u32 tcp_cookie_time(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) u64 val = get_jiffies_64();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) do_div(val, TCP_SYNCOOKIE_PERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) u16 *mssp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) u64 cookie_init_timestamp(struct request_sock *req, u64 now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) bool cookie_timestamp_decode(const struct net *net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) struct tcp_options_received *opt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) bool cookie_ecn_ok(const struct tcp_options_received *opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) const struct net *net, const struct dst_entry *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) /* From net/ipv6/syncookies.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) u32 cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) const struct tcphdr *th, u16 *mssp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /* tcp_output.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) int nonagle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) void tcp_retransmit_timer(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) void tcp_xmit_retransmit_queue(struct sock *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) void tcp_simple_retransmit(struct sock *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) void tcp_enter_recovery(struct sock *sk, bool ece_ack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) int tcp_trim_head(struct sock *, struct sk_buff *, u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) enum tcp_queue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) TCP_FRAG_IN_WRITE_QUEUE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) TCP_FRAG_IN_RTX_QUEUE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) struct sk_buff *skb, u32 len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) unsigned int mss_now, gfp_t gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) void tcp_send_probe0(struct sock *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) void tcp_send_partial(struct sock *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) int tcp_write_wakeup(struct sock *, int mib);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) void tcp_send_fin(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) void tcp_send_active_reset(struct sock *sk, gfp_t priority);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) int tcp_send_synack(struct sock *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) void tcp_push_one(struct sock *, unsigned int mss_now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) void __tcp_send_ack(struct sock *sk, u32 rcv_nxt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) void tcp_send_ack(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) void tcp_send_delayed_ack(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) void tcp_send_loss_probe(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) void tcp_skb_collapse_tstamp(struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) const struct sk_buff *next_skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) /* tcp_input.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) void tcp_rearm_rto(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) void tcp_reset(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) void tcp_fin(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) /* tcp_timer.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) void tcp_init_xmit_timers(struct sock *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) static inline void tcp_clear_xmit_timers(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) __sock_put(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) __sock_put(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) inet_csk_clear_xmit_timers(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) unsigned int tcp_current_mss(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) /* Bound MSS / TSO packet size with the half of the window */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) int cutoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) /* When peer uses tiny windows, there is no use in packetizing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * to sub-MSS pieces for the sake of SWS or making sure there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * are enough packets in the pipe for fast recovery.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * On the other hand, for extremely large MSS devices, handling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) * smaller than MSS windows in this way does make sense.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (tp->max_window > TCP_MSS_DEFAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) cutoff = (tp->max_window >> 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) cutoff = tp->max_window;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (cutoff && pktsize > cutoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) return max_t(int, cutoff, 68U - tp->tcp_header_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) return pktsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) /* tcp.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) void tcp_get_info(struct sock *, struct tcp_info *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) /* Read 'sendfile()'-style from a TCP socket */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) sk_read_actor_t recv_actor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) void tcp_initialize_rcv_mss(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) int tcp_mtu_to_mss(struct sock *sk, int pmtu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) int tcp_mss_to_mtu(struct sock *sk, int mss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) void tcp_mtup_init(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) static inline void tcp_bound_rto(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) inet_csk(sk)->icsk_rto = TCP_RTO_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) /* mptcp hooks are only on the slow path */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (sk_is_mptcp((struct sock *)tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) tp->pred_flags = htonl((tp->tcp_header_len << 26) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) ntohl(TCP_FLAG_ACK) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) snd_wnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) static inline void tcp_fast_path_on(struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) static inline void tcp_fast_path_check(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) struct tcp_sock *tp = tcp_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) tp->rcv_wnd &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) !tp->urg_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) tcp_fast_path_on(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) /* Compute the actual rto_min value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) static inline u32 tcp_rto_min(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) const struct dst_entry *dst = __sk_dst_get(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) u32 rto_min = inet_csk(sk)->icsk_rto_min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) return rto_min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) static inline u32 tcp_rto_min_us(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) return jiffies_to_usecs(tcp_rto_min(sk));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) return dst_metric_locked(dst, RTAX_CC_ALGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) /* Minimum RTT in usec. ~0 means not available. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) return minmax_get(&tp->rtt_min);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) /* Compute the actual receive window we are currently advertising.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) * Rcv_nxt can be after the window if our peer push more data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) * than the offered window.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) static inline u32 tcp_receive_window(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) if (win < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) win = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) return (u32) win;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) /* Choose a new window, without checks for shrinking, and without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * scaling applied to the result. The caller does these things
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * if necessary. This is a "raw" window selection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) u32 __tcp_select_window(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) void tcp_send_window_probe(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) /* TCP uses 32bit jiffies to save some space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * Note that this is different from tcp_time_stamp, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * historically has been the same until linux-4.13.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) #define tcp_jiffies32 ((u32)jiffies)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * Deliver a 32bit value for TCP timestamp option (RFC 7323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * It is no longer tied to jiffies, but to 1 ms clock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * Note: double check if you want to use tcp_jiffies32 instead of this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) #define TCP_TS_HZ 1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) static inline u64 tcp_clock_ns(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) return ktime_get_ns();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) static inline u64 tcp_clock_us(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) return div_u64(tcp_clock_ns(), NSEC_PER_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) /* This should only be used in contexts where tp->tcp_mstamp is up to date */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) static inline u32 tcp_time_stamp(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) /* Convert a nsec timestamp into TCP TSval timestamp (ms based currently) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) static inline u32 tcp_ns_to_ts(u64 ns)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) return div_u64(ns, NSEC_PER_SEC / TCP_TS_HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) /* Could use tcp_clock_us() / 1000, but this version uses a single divide */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) static inline u32 tcp_time_stamp_raw(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) return tcp_ns_to_ts(tcp_clock_ns());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) void tcp_mstamp_refresh(struct tcp_sock *tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) return max_t(s64, t1 - t0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) return tcp_ns_to_ts(skb->skb_mstamp_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) /* provide the departure time in us unit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) #define tcp_flag_byte(th) (((u_int8_t *)th)[13])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) #define TCPHDR_FIN 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) #define TCPHDR_SYN 0x02
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) #define TCPHDR_RST 0x04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) #define TCPHDR_PSH 0x08
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) #define TCPHDR_ACK 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) #define TCPHDR_URG 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) #define TCPHDR_ECE 0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) #define TCPHDR_CWR 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) /* This is what the send packet queuing engine uses to pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) * TCP per-packet control information to the transmission code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) * We also store the host-order sequence numbers in here too.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) * This is 44 bytes if IPV6 is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) struct tcp_skb_cb {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) __u32 seq; /* Starting sequence number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) __u32 end_seq; /* SEQ + FIN + SYN + datalen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) /* Note : tcp_tw_isn is used in input path only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * (isn chosen by tcp_timewait_state_process())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * tcp_gso_segs/size are used in write queue only,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * cf tcp_skb_pcount()/tcp_skb_mss()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) __u32 tcp_tw_isn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) u16 tcp_gso_segs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) u16 tcp_gso_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) __u8 tcp_flags; /* TCP header flags. (tcp[13]) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) __u8 sacked; /* State flags for SACK. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) #define TCPCB_SACKED_ACKED 0x01 /* SKB ACK'd by a SACK block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) #define TCPCB_SACKED_RETRANS 0x02 /* SKB retransmitted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) #define TCPCB_LOST 0x04 /* SKB is lost */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) #define TCPCB_TAGBITS 0x07 /* All tag bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) #define TCPCB_REPAIRED 0x10 /* SKB repaired (no skb_mstamp_ns) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) #define TCPCB_EVER_RETRANS 0x80 /* Ever retransmitted frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) #define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) TCPCB_REPAIRED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) __u8 txstamp_ack:1, /* Record TX timestamp for ack? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) eor:1, /* Is skb MSG_EOR marked? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) has_rxtstamp:1, /* SKB has a RX timestamp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) unused:5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) __u32 ack_seq; /* Sequence number ACK'd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) /* There is space for up to 24 bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) __u32 in_flight:30,/* Bytes in flight at transmit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) is_app_limited:1, /* cwnd not fully used? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) unused:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) /* pkts S/ACKed so far upon tx of skb, incl retrans: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) __u32 delivered;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) /* start of send pipeline phase */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) u64 first_tx_mstamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) /* when we reached the "delivered" count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) u64 delivered_mstamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) } tx; /* only used for outgoing skbs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) struct inet_skb_parm h4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) #if IS_ENABLED(CONFIG_IPV6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) struct inet6_skb_parm h6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) } header; /* For incoming skbs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) __u32 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) struct sock *sk_redir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) void *data_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) } bpf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) static inline void bpf_compute_data_end_sk_skb(struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) TCP_SKB_CB(skb)->bpf.data_end = skb->data + skb_headlen(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) static inline bool tcp_skb_bpf_ingress(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) return TCP_SKB_CB(skb)->bpf.flags & BPF_F_INGRESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) static inline struct sock *tcp_skb_bpf_redirect_fetch(struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) return TCP_SKB_CB(skb)->bpf.sk_redir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) static inline void tcp_skb_bpf_redirect_clear(struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) TCP_SKB_CB(skb)->bpf.sk_redir = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) extern const struct inet_connection_sock_af_ops ipv4_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) #if IS_ENABLED(CONFIG_IPV6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) /* This is the variant of inet6_iif() that must be used by TCP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * as TCP moves IP6CB into a different location in skb->cb[]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) static inline int tcp_v6_iif(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) return TCP_SKB_CB(skb)->header.h6.iif;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) /* TCP_SKB_CB reference means this can not be used from early demux */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) static inline int tcp_v6_sdif(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) return TCP_SKB_CB(skb)->header.h6.iif;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) extern const struct inet_connection_sock_af_ops ipv6_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) INDIRECT_CALLABLE_DECLARE(void tcp_v6_early_demux(struct sk_buff *skb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) /* TCP_SKB_CB reference means this can not be used from early demux */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) static inline int tcp_v4_sdif(struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) return TCP_SKB_CB(skb)->header.h4.iif;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) /* Due to TSO, an SKB can be composed of multiple actual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) * packets. To keep these tracked properly, we use this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) static inline int tcp_skb_pcount(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) return TCP_SKB_CB(skb)->tcp_gso_segs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) TCP_SKB_CB(skb)->tcp_gso_segs = segs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) TCP_SKB_CB(skb)->tcp_gso_segs += segs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) static inline int tcp_skb_mss(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) return TCP_SKB_CB(skb)->tcp_gso_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) return likely(!TCP_SKB_CB(skb)->eor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) const struct sk_buff *from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) return likely(tcp_skb_can_collapse_to(to) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) mptcp_skb_can_collapse(to, from));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) /* Events passed to congestion control interface */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) enum tcp_ca_event {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) CA_EVENT_TX_START, /* first transmit when no packets in flight */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) CA_EVENT_CWND_RESTART, /* congestion window restart */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) CA_EVENT_LOSS, /* loss timeout */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) /* Information about inbound ACK, passed to cong_ops->in_ack_event() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) enum tcp_ca_ack_event_flags {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * Interface for adding new TCP congestion control handlers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) #define TCP_CA_NAME_MAX 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) #define TCP_CA_MAX 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) #define TCP_CA_UNSPEC 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) #define TCP_CONG_NON_RESTRICTED 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) /* Requires ECN/ECT set on all packets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) #define TCP_CONG_NEEDS_ECN 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) #define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) union tcp_cc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) struct ack_sample {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) u32 pkts_acked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) s32 rtt_us;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) u32 in_flight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) /* A rate sample measures the number of (original/retransmitted) data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) * packets delivered "delivered" over an interval of time "interval_us".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) * The tcp_rate.c code fills in the rate sample, and congestion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) * control modules that define a cong_control function to run at the end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * of ACK processing can optionally chose to consult this sample when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * setting cwnd and pacing rate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * A sample is invalid if "delivered" or "interval_us" is negative.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) struct rate_sample {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) u64 prior_mstamp; /* starting timestamp for interval */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) u32 prior_delivered; /* tp->delivered at "prior_mstamp" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) s32 delivered; /* number of packets delivered over interval */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) long interval_us; /* time for tp->delivered to incr "delivered" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) u32 snd_interval_us; /* snd interval for delivered packets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) u32 rcv_interval_us; /* rcv interval for delivered packets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) long rtt_us; /* RTT of last (S)ACKed packet (or -1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) int losses; /* number of packets marked lost upon ACK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) u32 prior_in_flight; /* in flight before this ACK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) bool is_app_limited; /* is sample from packet with bubble in pipe? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) bool is_retrans; /* is sample from retransmission? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) bool is_ack_delayed; /* is this (likely) a delayed ACK? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) struct tcp_congestion_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) u32 key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) u32 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) /* initialize private data (optional) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) void (*init)(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) /* cleanup private data (optional) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) void (*release)(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) /* return slow start threshold (required) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) u32 (*ssthresh)(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) /* do new cwnd calculation (required) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) /* call before changing ca_state (optional) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) void (*set_state)(struct sock *sk, u8 new_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) /* call when cwnd event occurs (optional) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) /* call when ack arrives (optional) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) void (*in_ack_event)(struct sock *sk, u32 flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) /* new value of cwnd after loss (required) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) u32 (*undo_cwnd)(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) /* hook for packet ack accounting (optional) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) /* override sysctl_tcp_min_tso_segs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) u32 (*min_tso_segs)(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) /* returns the multiplier used in tcp_sndbuf_expand (optional) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) u32 (*sndbuf_expand)(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) /* call when packets are delivered to update cwnd and pacing rate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * after all the ca_state processing. (optional)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) /* get info for inet_diag (optional) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) union tcp_cc_info *info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) char name[TCP_CA_NAME_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) struct module *owner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) int tcp_register_congestion_control(struct tcp_congestion_ops *type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) void tcp_assign_congestion_control(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) void tcp_init_congestion_control(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) void tcp_cleanup_congestion_control(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) int tcp_set_default_congestion_control(struct net *net, const char *name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) void tcp_get_default_congestion_control(struct net *net, char *name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) void tcp_get_available_congestion_control(char *buf, size_t len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) void tcp_get_allowed_congestion_control(char *buf, size_t len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) int tcp_set_allowed_congestion_control(char *allowed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) bool cap_net_admin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) u32 tcp_reno_ssthresh(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) u32 tcp_reno_undo_cwnd(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) extern struct tcp_congestion_ops tcp_reno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) struct tcp_congestion_ops *tcp_ca_find(const char *name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) #ifdef CONFIG_INET
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) char *tcp_ca_get_name_by_key(u32 key, char *buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) static inline bool tcp_ca_needs_ecn(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) const struct inet_connection_sock *icsk = inet_csk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) struct inet_connection_sock *icsk = inet_csk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) if (icsk->icsk_ca_ops->set_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) icsk->icsk_ca_ops->set_state(sk, ca_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) icsk->icsk_ca_state = ca_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) const struct inet_connection_sock *icsk = inet_csk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) if (icsk->icsk_ca_ops->cwnd_event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) icsk->icsk_ca_ops->cwnd_event(sk, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) /* From tcp_rate.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) struct rate_sample *rs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) bool is_sack_reneg, struct rate_sample *rs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) void tcp_rate_check_app_limited(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) /* These functions determine how the current flow behaves in respect of SACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) * handling. SACK is negotiated with the peer, and therefore it can vary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) * between different flows.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) * tcp_is_sack - SACK enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) * tcp_is_reno - No SACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) static inline int tcp_is_sack(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) return likely(tp->rx_opt.sack_ok);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) static inline bool tcp_is_reno(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) return !tcp_is_sack(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) return tp->sacked_out + tp->lost_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) /* This determines how many packets are "in the network" to the best
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) * of our knowledge. In many cases it is conservative, but where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) * detailed information is available from the receiver (via SACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) * blocks etc.) we can make more aggressive calculations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) * Use this for decisions involving congestion control, use just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) * tp->packets_out to determine if the send queue is empty or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) * Read this equation as:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) * "Packets sent once on transmission queue" MINUS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) * "Packets left network, but not honestly ACKed yet" PLUS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) * "Packets fast retransmitted"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) #define TCP_INFINITE_SSTHRESH 0x7fffffff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) return tp->snd_cwnd < tp->snd_ssthresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) return (TCPF_CA_CWR | TCPF_CA_Recovery) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) (1 << inet_csk(sk)->icsk_ca_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) * The exception is cwnd reduction phase, when cwnd is decreasing towards
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) * ssthresh.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) static inline __u32 tcp_current_ssthresh(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) const struct tcp_sock *tp = tcp_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) if (tcp_in_cwnd_reduction(sk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) return tp->snd_ssthresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) return max(tp->snd_ssthresh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) ((tp->snd_cwnd >> 1) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) (tp->snd_cwnd >> 2)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) /* Use define here intentionally to get WARN_ON location shown at the caller */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) void tcp_enter_cwr(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) /* The maximum number of MSS of available cwnd for which TSO defers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) * sending if not using sysctl_tcp_tso_win_divisor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) return 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) /* Returns end sequence number of the receiver's advertised window */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) return tp->snd_una + tp->snd_wnd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) /* We follow the spirit of RFC2861 to validate cwnd but implement a more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) * flexible approach. The RFC suggests cwnd should not be raised unless
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) * it was fully used previously. And that's exactly what we do in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) * congestion avoidance mode. But in slow start we allow cwnd to grow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) * as long as the application has used half the cwnd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) * Example :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) * cwnd is 10 (IW10), but application sends 9 frames.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) * We allow cwnd to reach 18 when all frames are ACKed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) * This check is safe because it's as aggressive as slow start which already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) * risks 100% overshoot. The advantage is that we discourage application to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) * either send more filler packets or data to artificially blow up the cwnd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) * usage, and allow application-limited process to probe bw more aggressively.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) static inline bool tcp_is_cwnd_limited(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) const struct tcp_sock *tp = tcp_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) /* If in slow start, ensure cwnd grows to twice what was ACKed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) if (tcp_in_slow_start(tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) return tp->snd_cwnd < 2 * tp->max_packets_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) return tp->is_cwnd_limited;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) /* BBR congestion control needs pacing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) * Same remark for SO_MAX_PACING_RATE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) * sch_fq packet scheduler is efficiently handling pacing,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) * but is not always installed/used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) * Return true if TCP stack should pace packets itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) static inline bool tcp_needs_internal_pacing(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) /* Estimates in how many jiffies next packet for this flow can be sent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * Scheduling a retransmit timer too early would be silly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) static inline unsigned long tcp_pacing_delay(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) return delay > 0 ? nsecs_to_jiffies(delay) : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) static inline void tcp_reset_xmit_timer(struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) const int what,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) unsigned long when,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) const unsigned long max_when)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) max_when);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) /* Something is really bad, we could not queue an additional packet,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) * because qdisc is full or receiver sent a 0 window, or we are paced.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) * We do not want to add fuel to the fire, or abort too early,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) * so make sure the timer we arm now is at least 200ms in the future,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) * regardless of current icsk_rto value (as it could be ~2ms)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) static inline unsigned long tcp_probe0_base(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) /* Variant of inet_csk_rto_backoff() used for zero window probes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) static inline unsigned long tcp_probe0_when(const struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) unsigned long max_when)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) u64 when = (u64)tcp_probe0_base(sk) << inet_csk(sk)->icsk_backoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) return (unsigned long)min_t(u64, when, max_when);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) static inline void tcp_check_probe_timer(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) tcp_probe0_base(sk), TCP_RTO_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) tp->snd_wl1 = seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) tp->snd_wl1 = seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) * Calculate(/check) TCP checksum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) static inline __sum16 tcp_v4_check(int len, __be32 saddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) __be32 daddr, __wsum base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) static inline bool tcp_checksum_complete(struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) return !skb_csum_unnecessary(skb) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) __skb_checksum_complete(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) int tcp_filter(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) void tcp_set_state(struct sock *sk, int state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) void tcp_done(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) int tcp_abort(struct sock *sk, int err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) rx_opt->dsack = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) rx_opt->num_sacks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) void tcp_cwnd_restart(struct sock *sk, s32 delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) static inline void tcp_slow_start_after_idle_check(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) struct tcp_sock *tp = tcp_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) s32 delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) if (!sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle || tp->packets_out ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) ca_ops->cong_control)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) delta = tcp_jiffies32 - tp->lsndtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) if (delta > inet_csk(sk)->icsk_rto)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) tcp_cwnd_restart(sk, delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) /* Determine a window scaling and initial window to offer. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) void tcp_select_initial_window(const struct sock *sk, int __space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) __u32 mss, __u32 *rcv_wnd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) __u32 *window_clamp, int wscale_ok,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) __u8 *rcv_wscale, __u32 init_rcv_wnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) static inline int tcp_win_from_space(const struct sock *sk, int space)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) int tcp_adv_win_scale = sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) return tcp_adv_win_scale <= 0 ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) (space>>(-tcp_adv_win_scale)) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) space - (space>>tcp_adv_win_scale);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) /* Note: caller must be prepared to deal with negative returns */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) static inline int tcp_space(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) READ_ONCE(sk->sk_backlog.len) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) atomic_read(&sk->sk_rmem_alloc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) static inline int tcp_full_space(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) void tcp_cleanup_rbuf(struct sock *sk, int copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) /* We provision sk_rcvbuf around 200% of sk_rcvlowat.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) * If 87.5 % (7/8) of the space has been consumed, we want to override
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) * SO_RCVLOWAT constraint, since we are receiving skbs with too small
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) * len/truesize ratio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) static inline bool tcp_rmem_pressure(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) int rcvbuf, threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) if (tcp_under_memory_pressure(sk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) rcvbuf = READ_ONCE(sk->sk_rcvbuf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) threshold = rcvbuf - (rcvbuf >> 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) return atomic_read(&sk->sk_rmem_alloc) > threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) extern void tcp_openreq_init_rwin(struct request_sock *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) const struct sock *sk_listener,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) const struct dst_entry *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) void tcp_enter_memory_pressure(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) void tcp_leave_memory_pressure(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) static inline int keepalive_intvl_when(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) struct net *net = sock_net((struct sock *)tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) static inline int keepalive_time_when(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) struct net *net = sock_net((struct sock *)tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) static inline int keepalive_probes(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) struct net *net = sock_net((struct sock *)tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) const struct inet_connection_sock *icsk = &tp->inet_conn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) tcp_jiffies32 - tp->rcv_tstamp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) static inline int tcp_fin_time(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) const int rto = inet_csk(sk)->icsk_rto;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) if (fin_timeout < (rto << 2) - (rto >> 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) fin_timeout = (rto << 2) - (rto >> 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) return fin_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) int paws_win)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) if (unlikely(!time_before32(ktime_get_seconds(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) * then following tcp messages have valid values. Ignore 0 value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) * or else 'negative' tsval might forbid us to accept their packets.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) if (!rx_opt->ts_recent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) int rst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) if (tcp_paws_check(rx_opt, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) /* RST segments are not recommended to carry timestamp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) and, if they do, it is recommended to ignore PAWS because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) "their cleanup function should take precedence over timestamps."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) Certainly, it is mistake. It is necessary to understand the reasons
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) of this constraint to relax it: if peer reboots, clock may go
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) out-of-sync and half-open connections will not be reset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) Actually, the problem would be not existing if all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) the implementations followed draft about maintaining clock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) via reboots. Linux-2.2 DOES NOT!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) However, we can relax time bounds for RST segments to MSL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) if (rst && !time_before32(ktime_get_seconds(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) int mib_idx, u32 *last_oow_ack_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) static inline void tcp_mib_init(struct net *net)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) /* See RFC 2012 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) /* from STCP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) tp->lost_skb_hint = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) tcp_clear_retrans_hints_partial(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) tp->retransmit_skb_hint = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) union tcp_md5_addr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) struct in_addr a4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) #if IS_ENABLED(CONFIG_IPV6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) struct in6_addr a6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) /* - key database */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) struct tcp_md5sig_key {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) struct hlist_node node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) u8 keylen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) u8 family; /* AF_INET or AF_INET6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) u8 prefixlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) union tcp_md5_addr addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) int l3index; /* set if key added with L3 scope */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) u8 key[TCP_MD5SIG_MAXKEYLEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) /* - sock block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) struct tcp_md5sig_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) struct hlist_head head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) /* - pseudo header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) struct tcp4_pseudohdr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) __be32 saddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) __be32 daddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) __u8 pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) __u8 protocol;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) __be16 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) struct tcp6_pseudohdr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) struct in6_addr saddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) struct in6_addr daddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) __be32 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) __be32 protocol; /* including padding */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) union tcp_md5sum_block {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) struct tcp4_pseudohdr ip4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) #if IS_ENABLED(CONFIG_IPV6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) struct tcp6_pseudohdr ip6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) /* - pool: digest algorithm, hash description and scratch buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) struct tcp_md5sig_pool {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) struct ahash_request *md5_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) void *scratch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) /* - functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) const struct sock *sk, const struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) int family, u8 prefixlen, int l3index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) const u8 *newkey, u8 newkeylen, gfp_t gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) int family, u8 prefixlen, int l3index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) const struct sock *addr_sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) #ifdef CONFIG_TCP_MD5SIG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) #include <linux/jump_label.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) extern struct static_key_false tcp_md5_needed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) const union tcp_md5_addr *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) int family);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) static inline struct tcp_md5sig_key *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) tcp_md5_do_lookup(const struct sock *sk, int l3index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) const union tcp_md5_addr *addr, int family)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) if (!static_branch_unlikely(&tcp_md5_needed))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) return __tcp_md5_do_lookup(sk, l3index, addr, family);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) static inline struct tcp_md5sig_key *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) tcp_md5_do_lookup(const struct sock *sk, int l3index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) const union tcp_md5_addr *addr, int family)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) #define tcp_twsk_md5_key(twsk) NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) bool tcp_alloc_md5sig_pool(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) struct tcp_md5sig_pool *tcp_get_md5sig_pool(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) static inline void tcp_put_md5sig_pool(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) unsigned int header_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) int tcp_md5_hash_key(struct tcp_md5sig_pool *hp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) const struct tcp_md5sig_key *key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) /* From tcp_fastopen.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) struct tcp_fastopen_cookie *cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) struct tcp_fastopen_cookie *cookie, bool syn_lost,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) u16 try_exp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) struct tcp_fastopen_request {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) /* Fast Open cookie. Size 0 means a cookie request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) struct tcp_fastopen_cookie cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) struct msghdr *data; /* data in MSG_FASTOPEN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) int copied; /* queued in tcp_connect() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) struct ubuf_info *uarg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) void tcp_free_fastopen_req(struct tcp_sock *tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) void tcp_fastopen_destroy_cipher(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) void tcp_fastopen_ctx_destroy(struct net *net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) void *primary_key, void *backup_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) u64 *key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) struct request_sock *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) struct tcp_fastopen_cookie *foc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) const struct dst_entry *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) void tcp_fastopen_init_key_once(struct net *net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) struct tcp_fastopen_cookie *cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) #define TCP_FASTOPEN_KEY_MAX 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) #define TCP_FASTOPEN_KEY_BUF_LENGTH \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) /* Fastopen key context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) struct tcp_fastopen_context {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) siphash_key_t key[TCP_FASTOPEN_KEY_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) int num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) extern unsigned int sysctl_tcp_fastopen_blackhole_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) void tcp_fastopen_active_disable(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) bool tcp_fastopen_active_should_disable(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) /* Caller needs to wrap with rcu_read_(un)lock() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) struct tcp_fastopen_context *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) if (!ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) return ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) const struct tcp_fastopen_cookie *orig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) orig->len == foc->len &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) !memcmp(orig->val, foc->val, foc->len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) return ctx->num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) /* Latencies incurred by various limits for a sender. They are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) * chronograph-like stats that are mutually exclusive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) enum tcp_chrono {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) TCP_CHRONO_UNSPEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) __TCP_CHRONO_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) /* This helper is needed, because skb->tcp_tsorted_anchor uses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) * the same memory storage than skb->destructor/_skb_refdst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) skb->destructor = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) skb->_skb_refdst = 0UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) #define tcp_skb_tsorted_save(skb) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) unsigned long _save = skb->_skb_refdst; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) skb->_skb_refdst = 0UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) #define tcp_skb_tsorted_restore(skb) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) skb->_skb_refdst = _save; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) void tcp_write_queue_purge(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) return skb_rb_first(&sk->tcp_rtx_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) return skb_rb_last(&sk->tcp_rtx_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) return skb_peek(&sk->sk_write_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) return skb_peek_tail(&sk->sk_write_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) #define tcp_for_write_queue_from_safe(skb, tmp, sk) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) static inline struct sk_buff *tcp_send_head(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) return skb_peek(&sk->sk_write_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) static inline bool tcp_skb_is_last(const struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) return skb_queue_is_last(&sk->sk_write_queue, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) * @sk: socket
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) * Since the write queue can have a temporary empty skb in it,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) * we must not use "return skb_queue_empty(&sk->sk_write_queue)"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) static inline bool tcp_write_queue_empty(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) const struct tcp_sock *tp = tcp_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) return tp->write_seq == tp->snd_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) static inline bool tcp_rtx_queue_empty(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) __skb_queue_tail(&sk->sk_write_queue, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) /* Queue it, remembering where we must start sending. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) if (sk->sk_write_queue.next == skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) tcp_chrono_start(sk, TCP_CHRONO_BUSY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) /* Insert new before skb on the write queue of sk. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) static inline void tcp_insert_write_queue_before(struct sk_buff *new,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) __skb_queue_before(&sk->sk_write_queue, skb, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) tcp_skb_tsorted_anchor_cleanup(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) __skb_unlink(skb, &sk->sk_write_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) tcp_skb_tsorted_anchor_cleanup(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) list_del(&skb->tcp_tsorted_anchor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) tcp_rtx_queue_unlink(skb, sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) sk_wmem_free_skb(sk, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) static inline void tcp_push_pending_frames(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) if (tcp_send_head(sk)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) struct tcp_sock *tp = tcp_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) /* Start sequence of the skb just after the highest skb with SACKed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) * bit, valid only if sacked_out > 0 or when the caller has ensured
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) * validity by itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) if (!tp->sacked_out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) return tp->snd_una;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) if (tp->highest_sack == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) return tp->snd_nxt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) return TCP_SKB_CB(tp->highest_sack)->seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) tcp_sk(sk)->highest_sack = skb_rb_next(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) return tcp_sk(sk)->highest_sack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) static inline void tcp_highest_sack_reset(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) /* Called when old skb is about to be deleted and replaced by new skb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) static inline void tcp_highest_sack_replace(struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) struct sk_buff *old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) struct sk_buff *new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) if (old == tcp_highest_sack(sk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) tcp_sk(sk)->highest_sack = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) /* This helper checks if socket has IP_TRANSPARENT set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) static inline bool inet_sk_transparent(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) switch (sk->sk_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) case TCP_TIME_WAIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) return inet_twsk(sk)->tw_transparent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) case TCP_NEW_SYN_RECV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) return inet_rsk(inet_reqsk(sk))->no_srccheck;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) return inet_sk(sk)->transparent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) /* Determines whether this is a thin stream (which may suffer from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) * increased latency). Used to trigger latency-reducing mechanisms.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) /* /proc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) enum tcp_seq_states {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) TCP_SEQ_STATE_LISTENING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) TCP_SEQ_STATE_ESTABLISHED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) void tcp_seq_stop(struct seq_file *seq, void *v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) struct tcp_seq_afinfo {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) sa_family_t family;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) struct tcp_iter_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) struct seq_net_private p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) enum tcp_seq_states state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) struct sock *syn_wait_sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) struct tcp_seq_afinfo *bpf_seq_afinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) int bucket, offset, sbucket, num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) loff_t last_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) extern struct request_sock_ops tcp_request_sock_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) extern struct request_sock_ops tcp6_request_sock_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) void tcp_v4_destroy_sock(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) netdev_features_t features);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) int tcp_gro_complete(struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) struct net *net = sock_net((struct sock *)tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) /* @wake is one when sk_stream_write_space() calls us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) * This sends EPOLLOUT only if notsent_bytes is half the limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) * This mimics the strategy used in sock_def_write_space().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) static inline bool tcp_stream_memory_free(const struct sock *sk, int wake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) const struct tcp_sock *tp = tcp_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) u32 notsent_bytes = READ_ONCE(tp->write_seq) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) READ_ONCE(tp->snd_nxt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) #ifdef CONFIG_PROC_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) int tcp4_proc_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) void tcp4_proc_exit(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) int tcp_conn_request(struct request_sock_ops *rsk_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) const struct tcp_request_sock_ops *af_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) /* TCP af-specific functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) struct tcp_sock_af_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) #ifdef CONFIG_TCP_MD5SIG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) const struct sock *addr_sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) int (*calc_md5_hash)(char *location,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) const struct tcp_md5sig_key *md5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) const struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) const struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) int (*md5_parse)(struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) int optname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) sockptr_t optval,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) int optlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) struct tcp_request_sock_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) u16 mss_clamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) #ifdef CONFIG_TCP_MD5SIG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) const struct sock *addr_sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) int (*calc_md5_hash) (char *location,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) const struct tcp_md5sig_key *md5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) const struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) const struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) void (*init_req)(struct request_sock *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) const struct sock *sk_listener,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) #ifdef CONFIG_SYN_COOKIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) __u32 (*cookie_init_seq)(const struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) __u16 *mss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) struct dst_entry *(*route_req)(const struct sock *sk, struct flowi *fl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) const struct request_sock *req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) u32 (*init_seq)(const struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) struct flowi *fl, struct request_sock *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) struct tcp_fastopen_cookie *foc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) enum tcp_synack_type synack_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) struct sk_buff *syn_skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) #if IS_ENABLED(CONFIG_IPV6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) #ifdef CONFIG_SYN_COOKIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) const struct sock *sk, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) __u16 *mss)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) tcp_synq_overflow(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) return ops->cookie_init_seq(skb, mss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) const struct sock *sk, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) __u16 *mss)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) int tcpv4_offload_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) void tcp_v4_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) void tcp_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) /* tcp_recovery.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) u32 reo_wnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) extern bool tcp_rack_mark_lost(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) u64 xmit_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) extern void tcp_rack_reo_timeout(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) /* At how many usecs into the future should the RTO fire? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) static inline s64 tcp_rto_delta_us(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) const struct sk_buff *skb = tcp_rtx_queue_head(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) u32 rto = inet_csk(sk)->icsk_rto;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) * Save and compile IPv4 options, return a pointer to it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) struct ip_options_rcu *dopt = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) if (opt->optlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) int opt_size = sizeof(*dopt) + opt->optlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) dopt = kmalloc(opt_size, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) kfree(dopt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) dopt = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) return dopt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) /* locally generated TCP pure ACKs have skb->truesize == 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) * (check tcp_send_ack() in net/ipv4/tcp_output.c )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) * This is much faster than dissecting the packet to find out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) * (Think of GRE encapsulations, IPv4, IPv6, ...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) return skb->truesize == 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) skb->truesize = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) static inline int tcp_inq(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) struct tcp_sock *tp = tcp_sk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) int answ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) answ = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) } else if (sock_flag(sk, SOCK_URGINLINE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) !tp->urg_data ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) before(tp->urg_seq, tp->copied_seq) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) !before(tp->urg_seq, tp->rcv_nxt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) answ = tp->rcv_nxt - tp->copied_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) /* Subtract 1, if FIN was received */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) if (answ && sock_flag(sk, SOCK_DONE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) answ--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) answ = tp->urg_seq - tp->copied_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) return answ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) int tcp_peek_len(struct socket *sock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) u16 segs_in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) tp->segs_in += segs_in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) if (skb->len > tcp_hdrlen(skb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) tp->data_segs_in += segs_in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) * TCP listen path runs lockless.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) * We forced "struct sock" to be const qualified to make sure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) * we don't modify one of its field by mistake.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) * Here, we increment sk_drops which is an atomic_t, so we can safely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) * make sock writable again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) static inline void tcp_listendrop(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) atomic_inc(&((struct sock *)sk)->sk_drops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) * Interface for adding Upper Level Protocols over TCP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) #define TCP_ULP_NAME_MAX 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) #define TCP_ULP_MAX 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) struct tcp_ulp_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) /* initialize ulp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) int (*init)(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) /* update ulp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) void (*update)(struct sock *sk, struct proto *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) void (*write_space)(struct sock *sk));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) /* cleanup ulp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) void (*release)(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) /* diagnostic */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) int (*get_info)(const struct sock *sk, struct sk_buff *skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) size_t (*get_info_size)(const struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) /* clone ulp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) void (*clone)(const struct request_sock *req, struct sock *newsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) const gfp_t priority);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) char name[TCP_ULP_NAME_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) struct module *owner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) int tcp_register_ulp(struct tcp_ulp_ops *type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) void tcp_unregister_ulp(struct tcp_ulp_ops *type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) int tcp_set_ulp(struct sock *sk, const char *name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) void tcp_get_available_ulp(char *buf, size_t len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) void tcp_cleanup_ulp(struct sock *sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) void tcp_update_ulp(struct sock *sk, struct proto *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) void (*write_space)(struct sock *sk));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) #define MODULE_ALIAS_TCP_ULP(name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) __MODULE_INFO(alias, alias_userspace, name); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) struct sk_msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) struct sk_psock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) #ifdef CONFIG_BPF_STREAM_PARSER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) #endif /* CONFIG_BPF_STREAM_PARSER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) #ifdef CONFIG_NET_SOCK_MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) int flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) struct msghdr *msg, int len, int flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) #endif /* CONFIG_NET_SOCK_MSG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) #ifdef CONFIG_CGROUP_BPF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) unsigned int end_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) skops->skb = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) skops->skb_data_end = skb->data + end_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) unsigned int end_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) /* Call BPF_SOCK_OPS program that returns an int. If the return value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) * is < 0, then the BPF op failed (for example if the loaded BPF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) * program does not support the chosen operation or there is no BPF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) * program loaded).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) #ifdef CONFIG_BPF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) struct bpf_sock_ops_kern sock_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) if (sk_fullsock(sk)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) sock_ops.is_fullsock = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) sock_owned_by_me(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) sock_ops.sk = sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) sock_ops.op = op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) if (nargs > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) memcpy(sock_ops.args, args, nargs * sizeof(*args));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) ret = sock_ops.reply;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) ret = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) u32 args[2] = {arg1, arg2};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) return tcp_call_bpf(sk, op, 2, args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) u32 arg3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) u32 args[3] = {arg1, arg2, arg3};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) return tcp_call_bpf(sk, op, 3, args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) u32 arg3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) static inline u32 tcp_timeout_init(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) int timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) if (timeout <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) timeout = TCP_TIMEOUT_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) return timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) int rwnd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) if (rwnd < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) rwnd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) return rwnd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) static inline void tcp_bpf_rtt(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) #if IS_ENABLED(CONFIG_SMC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) extern struct static_key_false tcp_have_smc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) #if IS_ENABLED(CONFIG_TLS_DEVICE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) void clean_acked_data_enable(struct inet_connection_sock *icsk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) void (*cad)(struct sock *sk, u32 ack_seq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) void clean_acked_data_disable(struct inet_connection_sock *icsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) void clean_acked_data_flush(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) static inline void tcp_add_tx_delay(struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) const struct tcp_sock *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) if (static_branch_unlikely(&tcp_tx_delay_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) /* Compute Earliest Departure Time for some control packets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) static inline u64 tcp_transmit_time(const struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) if (static_branch_unlikely(&tcp_tx_delay_enabled)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) #endif /* _TCP_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags