Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #include <net/tcp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #include <net/strparser.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) #include <net/xfrm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <net/esp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <net/espintcp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <linux/skmsg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <net/inet_common.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #if IS_ENABLED(CONFIG_IPV6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <net/ipv6_stubs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 			  struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 	    !sk_rmem_schedule(sk, skb, skb->truesize)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 		XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 		kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	skb_set_owner_r(skb, sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	memset(skb->cb, 0, sizeof(skb->cb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	skb_queue_tail(&ctx->ike_queue, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	ctx->saved_data_ready(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) static void handle_esp(struct sk_buff *skb, struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	struct tcp_skb_cb *tcp_cb = (struct tcp_skb_cb *)skb->cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	skb_reset_transport_header(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	/* restore IP CB, we need at least IP6CB->nhoff */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	memmove(skb->cb, &tcp_cb->header, sizeof(tcp_cb->header));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) #if IS_ENABLED(CONFIG_IPV6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	if (sk->sk_family == AF_INET6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 		ipv6_stub->xfrm6_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 		xfrm4_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) static void espintcp_rcv(struct strparser *strp, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	struct espintcp_ctx *ctx = container_of(strp, struct espintcp_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 						strp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	struct strp_msg *rxm = strp_msg(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	int len = rxm->full_len - 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	u32 nonesp_marker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	/* keepalive packet? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	if (unlikely(len == 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 		u8 data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 		err = skb_copy_bits(skb, rxm->offset + 2, &data, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 		if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 			XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 			kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 			return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 		if (data == 0xff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 			kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 			return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	/* drop other short messages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	if (unlikely(len <= sizeof(nonesp_marker))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 		XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 		kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	err = skb_copy_bits(skb, rxm->offset + 2, &nonesp_marker,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 			    sizeof(nonesp_marker));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 		XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	/* remove header, leave non-ESP marker/SPI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	if (!__pskb_pull(skb, rxm->offset + 2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 		XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 		kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	if (pskb_trim(skb, rxm->full_len - 2) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 		XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 		kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	if (nonesp_marker == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 		handle_nonesp(ctx, skb, strp->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 		handle_esp(skb, strp->sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) static int espintcp_parse(struct strparser *strp, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	struct strp_msg *rxm = strp_msg(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	__be16 blen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	u16 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	if (skb->len < rxm->offset + 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	err = skb_copy_bits(skb, rxm->offset, &blen, sizeof(blen));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 		return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	len = be16_to_cpu(blen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	if (len < 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static int espintcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 			    int nonblock, int flags, int *addr_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	int copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	int off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	flags |= nonblock ? MSG_DONTWAIT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	skb = __skb_recv_datagram(sk, &ctx->ike_queue, flags, &off, &err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	if (!skb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 		if (err == -EAGAIN && sk->sk_shutdown & RCV_SHUTDOWN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 			return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 		return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	copied = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	if (copied > skb->len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 		copied = skb->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	else if (copied < skb->len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 		msg->msg_flags |= MSG_TRUNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	err = skb_copy_datagram_msg(skb, 0, msg, copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	if (unlikely(err)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 		kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 		return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	if (flags & MSG_TRUNC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 		copied = skb->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	return copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) int espintcp_queue_out(struct sock *sk, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	if (skb_queue_len(&ctx->out_queue) >= netdev_max_backlog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 		return -ENOBUFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	__skb_queue_tail(&ctx->out_queue, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) EXPORT_SYMBOL_GPL(espintcp_queue_out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) /* espintcp length field is 2B and length includes the length field's size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #define MAX_ESPINTCP_MSG (((1 << 16) - 1) - 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) static int espintcp_sendskb_locked(struct sock *sk, struct espintcp_msg *emsg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 				   int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 		int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		ret = skb_send_sock_locked(sk, emsg->skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 					   emsg->offset, emsg->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 		if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 			return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 		emsg->len -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 		emsg->offset += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	} while (emsg->len > 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	kfree_skb(emsg->skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	memset(emsg, 0, sizeof(*emsg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) static int espintcp_sendskmsg_locked(struct sock *sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 				     struct espintcp_msg *emsg, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	struct sk_msg *skmsg = &emsg->skmsg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	int done = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	flags |= MSG_SENDPAGE_NOTLAST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	sg = &skmsg->sg.data[skmsg->sg.start];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 		size_t size = sg->length - emsg->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 		int offset = sg->offset + emsg->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 		struct page *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 		emsg->offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 		if (sg_is_last(sg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 			flags &= ~MSG_SENDPAGE_NOTLAST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 		p = sg_page(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 		ret = do_tcp_sendpages(sk, p, offset, size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 		if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 			emsg->offset = offset - sg->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 			skmsg->sg.start += done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 			return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 		if (ret != size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 			offset += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 			size -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 			goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		done++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 		put_page(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		sk_mem_uncharge(sk, sg->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 		sg = sg_next(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	} while (sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	memset(emsg, 0, sizeof(*emsg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) static int espintcp_push_msgs(struct sock *sk, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	struct espintcp_msg *emsg = &ctx->partial;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	if (!emsg->len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	if (ctx->tx_running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 		return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	ctx->tx_running = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	if (emsg->skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 		err = espintcp_sendskb_locked(sk, emsg, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 		err = espintcp_sendskmsg_locked(sk, emsg, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	if (err == -EAGAIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		ctx->tx_running = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 		return flags & MSG_DONTWAIT ? -EAGAIN : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 		memset(emsg, 0, sizeof(*emsg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	ctx->tx_running = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) int espintcp_push_skb(struct sock *sk, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	struct espintcp_msg *emsg = &ctx->partial;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	unsigned int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	int offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	if (sk->sk_state != TCP_ESTABLISHED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 		kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 		return -ECONNRESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	offset = skb_transport_offset(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	len = skb->len - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	espintcp_push_msgs(sk, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	if (emsg->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 		kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 		return -ENOBUFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	skb_set_owner_w(skb, sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	emsg->offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	emsg->len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	emsg->skb = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	espintcp_push_msgs(sk, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) EXPORT_SYMBOL_GPL(espintcp_push_skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static int espintcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	struct espintcp_msg *emsg = &ctx->partial;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	struct iov_iter pfx_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	struct kvec pfx_iov = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	size_t msglen = size + 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	char buf[2] = {0};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	int err, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	if (msg->msg_flags & ~MSG_DONTWAIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 		return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	if (size > MAX_ESPINTCP_MSG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 		return -EMSGSIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	if (msg->msg_controllen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 		return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 	lock_sock(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 		if (err != -EAGAIN || !(msg->msg_flags & MSG_DONTWAIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 			err = -ENOBUFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 		goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	sk_msg_init(&emsg->skmsg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 		/* only -ENOMEM is possible since we don't coalesce */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 		err = sk_msg_alloc(sk, &emsg->skmsg, msglen, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 		if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 		err = sk_stream_wait_memory(sk, &timeo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 		if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 			goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 	*((__be16 *)buf) = cpu_to_be16(msglen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 	pfx_iov.iov_base = buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 	pfx_iov.iov_len = sizeof(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	iov_iter_kvec(&pfx_iter, WRITE, &pfx_iov, 1, pfx_iov.iov_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	err = sk_msg_memcopy_from_iter(sk, &pfx_iter, &emsg->skmsg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 				       pfx_iov.iov_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 	if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 		goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, &emsg->skmsg, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 		goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 	end = emsg->skmsg.sg.end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	emsg->len = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	sk_msg_iter_var_prev(end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	sg_mark_end(sk_msg_elem(&emsg->skmsg, end));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 	tcp_rate_check_app_limited(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 	err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	/* this message could be partially sent, keep it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	release_sock(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 	return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 	sk_msg_free(sk, &emsg->skmsg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	memset(emsg, 0, sizeof(*emsg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 	release_sock(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 	return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) static struct proto espintcp_prot __ro_after_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) static struct proto_ops espintcp_ops __ro_after_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) static struct proto espintcp6_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) static struct proto_ops espintcp6_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) static DEFINE_MUTEX(tcpv6_prot_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) static void espintcp_data_ready(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 	strp_data_ready(&ctx->strp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) static void espintcp_tx_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 	struct espintcp_ctx *ctx = container_of(work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 						struct espintcp_ctx, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	struct sock *sk = ctx->strp.sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 	lock_sock(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 	if (!ctx->tx_running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) 		espintcp_push_msgs(sk, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) 	release_sock(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) static void espintcp_write_space(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 	schedule_work(&ctx->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 	ctx->saved_write_space(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) static void espintcp_destruct(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) 	ctx->saved_destruct(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 	kfree(ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) bool tcp_is_ulp_esp(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 	return sk->sk_prot == &espintcp_prot || sk->sk_prot == &espintcp6_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) EXPORT_SYMBOL_GPL(tcp_is_ulp_esp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) static void build_protos(struct proto *espintcp_prot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 			 struct proto_ops *espintcp_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 			 const struct proto *orig_prot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 			 const struct proto_ops *orig_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) static int espintcp_init_sk(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 	struct inet_connection_sock *icsk = inet_csk(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) 	struct strp_callbacks cb = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) 		.rcv_msg = espintcp_rcv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) 		.parse_msg = espintcp_parse,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) 	};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 	struct espintcp_ctx *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) 	/* sockmap is not compatible with espintcp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) 	if (sk->sk_user_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) 		return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) 	if (!ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) 		return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) 	err = strp_init(&ctx->strp, sk, &cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) 	if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) 		goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) 	__sk_dst_reset(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) 	strp_check_rcv(&ctx->strp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) 	skb_queue_head_init(&ctx->ike_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) 	skb_queue_head_init(&ctx->out_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) 	if (sk->sk_family == AF_INET) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 		sk->sk_prot = &espintcp_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) 		sk->sk_socket->ops = &espintcp_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) 		mutex_lock(&tcpv6_prot_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) 		if (!espintcp6_prot.recvmsg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) 			build_protos(&espintcp6_prot, &espintcp6_ops, sk->sk_prot, sk->sk_socket->ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) 		mutex_unlock(&tcpv6_prot_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 		sk->sk_prot = &espintcp6_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 		sk->sk_socket->ops = &espintcp6_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 	ctx->saved_data_ready = sk->sk_data_ready;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) 	ctx->saved_write_space = sk->sk_write_space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 	ctx->saved_destruct = sk->sk_destruct;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) 	sk->sk_data_ready = espintcp_data_ready;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) 	sk->sk_write_space = espintcp_write_space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) 	sk->sk_destruct = espintcp_destruct;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) 	rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) 	INIT_WORK(&ctx->work, espintcp_tx_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 	/* avoid using task_frag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) 	sk->sk_allocation = GFP_ATOMIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 	kfree(ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) 	return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) static void espintcp_release(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 	struct sk_buff_head queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) 	struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) 	__skb_queue_head_init(&queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 	skb_queue_splice_init(&ctx->out_queue, &queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) 	while ((skb = __skb_dequeue(&queue)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) 		espintcp_push_skb(sk, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) 	tcp_release_cb(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) static void espintcp_close(struct sock *sk, long timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) 	struct espintcp_msg *emsg = &ctx->partial;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) 	strp_stop(&ctx->strp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) 	sk->sk_prot = &tcp_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) 	barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) 	cancel_work_sync(&ctx->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) 	strp_done(&ctx->strp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) 	skb_queue_purge(&ctx->out_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) 	skb_queue_purge(&ctx->ike_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) 	if (emsg->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) 		if (emsg->skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) 			kfree_skb(emsg->skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) 			sk_msg_free(sk, &emsg->skmsg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) 	tcp_close(sk, timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) static __poll_t espintcp_poll(struct file *file, struct socket *sock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) 			      poll_table *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) 	__poll_t mask = datagram_poll(file, sock, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) 	struct sock *sk = sock->sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) 	struct espintcp_ctx *ctx = espintcp_getctx(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) 	if (!skb_queue_empty(&ctx->ike_queue))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) 		mask |= EPOLLIN | EPOLLRDNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) 	return mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) static void build_protos(struct proto *espintcp_prot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) 			 struct proto_ops *espintcp_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) 			 const struct proto *orig_prot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) 			 const struct proto_ops *orig_ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) 	memcpy(espintcp_prot, orig_prot, sizeof(struct proto));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) 	memcpy(espintcp_ops, orig_ops, sizeof(struct proto_ops));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) 	espintcp_prot->sendmsg = espintcp_sendmsg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) 	espintcp_prot->recvmsg = espintcp_recvmsg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) 	espintcp_prot->close = espintcp_close;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) 	espintcp_prot->release_cb = espintcp_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) 	espintcp_ops->poll = espintcp_poll;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) static struct tcp_ulp_ops espintcp_ulp __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) 	.name = "espintcp",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) 	.owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) 	.init = espintcp_init_sk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) void __init espintcp_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) 	build_protos(&espintcp_prot, &espintcp_ops, &tcp_prot, &inet_stream_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) 	tcp_register_ulp(&espintcp_ulp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }