1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* INET An implementation of the TCP/IP protocol suite for the LINUX
4	* operating system. INET is implemented using the BSD Socket
5	* interface as the means of communication with the user level.
6	*
7	* Implementation of the Transmission Control Protocol(TCP).
8	*
9	* Authors: Ross Biro
10	* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11	* Mark Evans, <evansmp@uhura.aston.ac.uk>
12	* Corey Minyard <wf-rch!minyard@relay.EU.net>
13	* Florian La Roche, <flla@stud.uni-sb.de>
14	* Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15	* Linus Torvalds, <torvalds@cs.helsinki.fi>
16	* Alan Cox, <gw4pts@gw4pts.ampr.org>
17	* Matthew Dillon, <dillon@apollo.west.oic.com>
18	* Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19	* Jorge Cwik, <jorge@laser.satlink.net>
20	*/
21
22	/*
23	* Changes:
24	* Pedro Roque : Fast Retransmit/Recovery.
25	* Two receive queues.
26	* Retransmit queue handled by TCP.
27	* Better retransmit timer handling.
28	* New congestion avoidance.
29	* Header prediction.
30	* Variable renaming.
31	*
32	* Eric : Fast Retransmit.
33	* Randy Scott : MSS option defines.
34	* Eric Schenk : Fixes to slow start algorithm.
35	* Eric Schenk : Yet another double ACK bug.
36	* Eric Schenk : Delayed ACK bug fixes.
37	* Eric Schenk : Floyd style fast retrans war avoidance.
38	* David S. Miller : Don't allow zero congestion window.
39	* Eric Schenk : Fix retransmitter so that it sends
40	* next packet on ack of previous packet.
41	* Andi Kleen : Moved open_request checking here
42	* and process RSTs for open_requests.
43	* Andi Kleen : Better prune_queue, and other fixes.
44	* Andrey Savochkin: Fix RTT measurements in the presence of
45	* timestamps.
46	* Andrey Savochkin: Check sequence numbers correctly when
47	* removing SACKs due to in sequence incoming
48	* data segments.
49	* Andi Kleen: Make sure we never ack data there is not
50	* enough room for. Also make this condition
51	* a fatal error if it might still happen.
52	* Andi Kleen: Add tcp_measure_rcv_mss to make
53	* connections with MSS<min(MTU,ann. MSS)
54	* work without delayed acks.
55	* Andi Kleen: Process packets with PSH set in the
56	* fast path.
57	* J Hadi Salim: ECN support
58	* Andrei Gurtov,
59	* Pasi Sarolahti,
60	* Panu Kuhlberg: Experimental audit of TCP (re)transmission
61	* engine. Lots of bugs are found.
62	* Pasi Sarolahti: F-RTO for dealing with spurious RTOs
63	*/
64
65	#define pr_fmt(fmt) "TCP: " fmt
66
67	#include <linux/mm.h>
68	#include <linux/slab.h>
69	#include <linux/module.h>
70	#include <linux/sysctl.h>
71	#include <linux/kernel.h>
72	#include <linux/prefetch.h>
73	#include <net/dst.h>
74	#include <net/tcp.h>
75	#include <net/inet_common.h>
76	#include <linux/ipsec.h>
77	#include <asm/unaligned.h>
78	#include <linux/errqueue.h>
79	#include <trace/events/tcp.h>
80	#include <linux/jump_label_ratelimit.h>
81	#include <net/busy_poll.h>
82	#include <net/mptcp.h>
83
84	int sysctl_tcp_max_orphans __read_mostly = NR_FILE;
85
86	#define FLAG_DATA 0x01 /* Incoming frame contained data. */
87	#define FLAG_WIN_UPDATE 0x02 /* Incoming ACK was a window update. */
88	#define FLAG_DATA_ACKED 0x04 /* This ACK acknowledged new data. */
89	#define FLAG_RETRANS_DATA_ACKED 0x08 /* "" "" some of which was retransmitted. */
90	#define FLAG_SYN_ACKED 0x10 /* This ACK acknowledged SYN. */
91	#define FLAG_DATA_SACKED 0x20 /* New SACK. */
92	#define FLAG_ECE 0x40 /* ECE in this ACK */
93	#define FLAG_LOST_RETRANS 0x80 /* This ACK marks some retransmission lost */
94	#define FLAG_SLOWPATH 0x100 /* Do not skip RFC checks for window update.*/
95	#define FLAG_ORIG_SACK_ACKED 0x200 /* Never retransmitted data are (s)acked */
96	#define FLAG_SND_UNA_ADVANCED 0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */
97	#define FLAG_DSACKING_ACK 0x800 /* SACK blocks contained D-SACK info */
98	#define FLAG_SET_XMIT_TIMER 0x1000 /* Set TLP or RTO timer */
99	#define FLAG_SACK_RENEGING 0x2000 /* snd_una advanced to a sacked seq */
100	#define FLAG_UPDATE_TS_RECENT 0x4000 /* tcp_replace_ts_recent() */
101	#define FLAG_NO_CHALLENGE_ACK 0x8000 /* do not call tcp_send_challenge_ack() */
102	#define FLAG_ACK_MAYBE_DELAYED 0x10000 /* Likely a delayed ACK */
103	#define FLAG_DSACK_TLP 0x20000 /* DSACK for tail loss probe */
104
105	#define FLAG_ACKED (FLAG_DATA_ACKED|FLAG_SYN_ACKED)
106	#define FLAG_NOT_DUP (FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED)
107	#define FLAG_CA_ALERT (FLAG_DATA_SACKED|FLAG_ECE|FLAG_DSACKING_ACK)
108	#define FLAG_FORWARD_PROGRESS (FLAG_ACKED|FLAG_DATA_SACKED)
109
110	#define TCP_REMNANT (TCP_FLAG_FIN|TCP_FLAG_URG|TCP_FLAG_SYN|TCP_FLAG_PSH)
111	#define TCP_HP_BITS (~(TCP_RESERVED_BITS|TCP_FLAG_PSH))
112
113	#define REXMIT_NONE 0 /* no loss recovery to do */
114	#define REXMIT_LOST 1 /* retransmit packets marked lost */
115	#define REXMIT_NEW 2 /* FRTO-style transmit of unsent/new packets */
116
117	#if IS_ENABLED(CONFIG_TLS_DEVICE)
118	static DEFINE_STATIC_KEY_DEFERRED_FALSE(clean_acked_data_enabled, HZ);
119
120	void clean_acked_data_enable(struct inet_connection_sock *icsk,
121	void (*cad)(struct sock *sk, u32 ack_seq))
122	{
123	icsk->icsk_clean_acked = cad;
124	static_branch_deferred_inc(&clean_acked_data_enabled);
125	}
126	EXPORT_SYMBOL_GPL(clean_acked_data_enable);
127
128	void clean_acked_data_disable(struct inet_connection_sock *icsk)
129	{
130	static_branch_slow_dec_deferred(&clean_acked_data_enabled);
131	icsk->icsk_clean_acked = NULL;
132	}
133	EXPORT_SYMBOL_GPL(clean_acked_data_disable);
134
135	void clean_acked_data_flush(void)
136	{
137	static_key_deferred_flush(&clean_acked_data_enabled);
138	}
139	EXPORT_SYMBOL_GPL(clean_acked_data_flush);
140	#endif
141
142	#ifdef CONFIG_CGROUP_BPF
143	static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
144	{
145	bool unknown_opt = tcp_sk(sk)->rx_opt.saw_unknown &&
146	BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
147	BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG);
148	bool parse_all_opt = BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
149	BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG);
150	struct bpf_sock_ops_kern sock_ops;
151
152	if (likely(!unknown_opt && !parse_all_opt))
153	return;
154
155	/* The skb will be handled in the
156	* bpf_skops_established() or
157	* bpf_skops_write_hdr_opt().
158	*/
159	switch (sk->sk_state) {
160	case TCP_SYN_RECV:
161	case TCP_SYN_SENT:
162	case TCP_LISTEN:
163	return;
164	}
165
166	sock_owned_by_me(sk);
167
168	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
169	sock_ops.op = BPF_SOCK_OPS_PARSE_HDR_OPT_CB;
170	sock_ops.is_fullsock = 1;
171	sock_ops.sk = sk;
172	bpf_skops_init_skb(skops: &sock_ops, skb, end_offset: tcp_hdrlen(skb));
173
174	BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
175	}
176
177	static void bpf_skops_established(struct sock *sk, int bpf_op,
178	struct sk_buff *skb)
179	{
180	struct bpf_sock_ops_kern sock_ops;
181
182	sock_owned_by_me(sk);
183
184	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
185	sock_ops.op = bpf_op;
186	sock_ops.is_fullsock = 1;
187	sock_ops.sk = sk;
188	/* sk with TCP_REPAIR_ON does not have skb in tcp_finish_connect */
189	if (skb)
190	bpf_skops_init_skb(skops: &sock_ops, skb, end_offset: tcp_hdrlen(skb));
191
192	BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
193	}
194	#else
195	static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
196	{
197	}
198
199	static void bpf_skops_established(struct sock *sk, int bpf_op,
200	struct sk_buff *skb)
201	{
202	}
203	#endif
204
205	static __cold void tcp_gro_dev_warn(const struct sock *sk, const struct sk_buff *skb,
206	unsigned int len)
207	{
208	struct net_device *dev;
209
210	rcu_read_lock();
211	dev = dev_get_by_index_rcu(net: sock_net(sk), ifindex: skb->skb_iif);
212	if (!dev || len >= READ_ONCE(dev->mtu))
213	pr_warn("%s: Driver has suspect GRO implementation, TCP performance may be compromised.\n",
214	dev ? dev->name : "Unknown driver");
215	rcu_read_unlock();
216	}
217
218	/* Adapt the MSS value used to make delayed ack decision to the
219	* real world.
220	*/
221	static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb)
222	{
223	struct inet_connection_sock *icsk = inet_csk(sk);
224	const unsigned int lss = icsk->icsk_ack.last_seg_size;
225	unsigned int len;
226
227	icsk->icsk_ack.last_seg_size = 0;
228
229	/* skb->len may jitter because of SACKs, even if peer
230	* sends good full-sized frames.
231	*/
232	len = skb_shinfo(skb)->gso_size ? : skb->len;
233	if (len >= icsk->icsk_ack.rcv_mss) {
234	/* Note: divides are still a bit expensive.
235	* For the moment, only adjust scaling_ratio
236	* when we update icsk_ack.rcv_mss.
237	*/
238	if (unlikely(len != icsk->icsk_ack.rcv_mss)) {
239	u64 val = (u64)skb->len << TCP_RMEM_TO_WIN_SCALE;
240
241	do_div(val, skb->truesize);
242	tcp_sk(sk)->scaling_ratio = val ? val : 1;
243	}
244	icsk->icsk_ack.rcv_mss = min_t(unsigned int, len,
245	tcp_sk(sk)->advmss);
246	/* Account for possibly-removed options */
247	DO_ONCE_LITE_IF(len > icsk->icsk_ack.rcv_mss + MAX_TCP_OPTION_SPACE,
248	tcp_gro_dev_warn, sk, skb, len);
249	/* If the skb has a len of exactly 1*MSS and has the PSH bit
250	* set then it is likely the end of an application write. So
251	* more data may not be arriving soon, and yet the data sender
252	* may be waiting for an ACK if cwnd-bound or using TX zero
253	* copy. So we set ICSK_ACK_PUSHED here so that
254	* tcp_cleanup_rbuf() will send an ACK immediately if the app
255	* reads all of the data and is not ping-pong. If len > MSS
256	* then this logic does not matter (and does not hurt) because
257	* tcp_cleanup_rbuf() will always ACK immediately if the app
258	* reads data and there is more than an MSS of unACKed data.
259	*/
260	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_PSH)
261	icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
262	} else {
263	/* Otherwise, we make more careful check taking into account,
264	* that SACKs block is variable.
265	*
266	* "len" is invariant segment length, including TCP header.
267	*/
268	len += skb->data - skb_transport_header(skb);
269	if (len >= TCP_MSS_DEFAULT + sizeof(struct tcphdr) ||
270	/* If PSH is not set, packet should be
271	* full sized, provided peer TCP is not badly broken.
272	* This observation (if it is correct 8)) allows
273	* to handle super-low mtu links fairly.
274	*/
275	(len >= TCP_MIN_MSS + sizeof(struct tcphdr) &&
276	!(tcp_flag_word(tcp_hdr(skb)) & TCP_REMNANT))) {
277	/* Subtract also invariant (if peer is RFC compliant),
278	* tcp header plus fixed timestamp option length.
279	* Resulting "len" is MSS free of SACK jitter.
280	*/
281	len -= tcp_sk(sk)->tcp_header_len;
282	icsk->icsk_ack.last_seg_size = len;
283	if (len == lss) {
284	icsk->icsk_ack.rcv_mss = len;
285	return;
286	}
287	}
288	if (icsk->icsk_ack.pending & ICSK_ACK_PUSHED)
289	icsk->icsk_ack.pending |= ICSK_ACK_PUSHED2;
290	icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
291	}
292	}
293
294	static void tcp_incr_quickack(struct sock *sk, unsigned int max_quickacks)
295	{
296	struct inet_connection_sock *icsk = inet_csk(sk);
297	unsigned int quickacks = tcp_sk(sk)->rcv_wnd / (2 * icsk->icsk_ack.rcv_mss);
298
299	if (quickacks == 0)
300	quickacks = 2;
301	quickacks = min(quickacks, max_quickacks);
302	if (quickacks > icsk->icsk_ack.quick)
303	icsk->icsk_ack.quick = quickacks;
304	}
305
306	static void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks)
307	{
308	struct inet_connection_sock *icsk = inet_csk(sk);
309
310	tcp_incr_quickack(sk, max_quickacks);
311	inet_csk_exit_pingpong_mode(sk);
312	icsk->icsk_ack.ato = TCP_ATO_MIN;
313	}
314
315	/* Send ACKs quickly, if "quick" count is not exhausted
316	* and the session is not interactive.
317	*/
318
319	static bool tcp_in_quickack_mode(struct sock *sk)
320	{
321	const struct inet_connection_sock *icsk = inet_csk(sk);
322	const struct dst_entry *dst = __sk_dst_get(sk);
323
324	return (dst && dst_metric(dst, RTAX_QUICKACK)) ||
325	(icsk->icsk_ack.quick && !inet_csk_in_pingpong_mode(sk));
326	}
327
328	static void tcp_ecn_queue_cwr(struct tcp_sock *tp)
329	{
330	if (tp->ecn_flags & TCP_ECN_OK)
331	tp->ecn_flags |= TCP_ECN_QUEUE_CWR;
332	}
333
334	static void tcp_ecn_accept_cwr(struct sock *sk, const struct sk_buff *skb)
335	{
336	if (tcp_hdr(skb)->cwr) {
337	tcp_sk(sk)->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
338
339	/* If the sender is telling us it has entered CWR, then its
340	* cwnd may be very low (even just 1 packet), so we should ACK
341	* immediately.
342	*/
343	if (TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq)
344	inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
345	}
346	}
347
348	static void tcp_ecn_withdraw_cwr(struct tcp_sock *tp)
349	{
350	tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
351	}
352
353	static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
354	{
355	struct tcp_sock *tp = tcp_sk(sk);
356
357	switch (TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK) {
358	case INET_ECN_NOT_ECT:
359	/* Funny extension: if ECT is not set on a segment,
360	* and we already seen ECT on a previous segment,
361	* it is probably a retransmit.
362	*/
363	if (tp->ecn_flags & TCP_ECN_SEEN)
364	tcp_enter_quickack_mode(sk, max_quickacks: 2);
365	break;
366	case INET_ECN_CE:
367	if (tcp_ca_needs_ecn(sk))
368	tcp_ca_event(sk, event: CA_EVENT_ECN_IS_CE);
369
370	if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) {
371	/* Better not delay acks, sender can have a very low cwnd */
372	tcp_enter_quickack_mode(sk, max_quickacks: 2);
373	tp->ecn_flags |= TCP_ECN_DEMAND_CWR;
374	}
375	tp->ecn_flags |= TCP_ECN_SEEN;
376	break;
377	default:
378	if (tcp_ca_needs_ecn(sk))
379	tcp_ca_event(sk, event: CA_EVENT_ECN_NO_CE);
380	tp->ecn_flags |= TCP_ECN_SEEN;
381	break;
382	}
383	}
384
385	static void tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
386	{
387	if (tcp_sk(sk)->ecn_flags & TCP_ECN_OK)
388	__tcp_ecn_check_ce(sk, skb);
389	}
390
391	static void tcp_ecn_rcv_synack(struct tcp_sock *tp, const struct tcphdr *th)
392	{
393	if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || th->cwr))
394	tp->ecn_flags &= ~TCP_ECN_OK;
395	}
396
397	static void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th)
398	{
399	if ((tp->ecn_flags & TCP_ECN_OK) && (!th->ece || !th->cwr))
400	tp->ecn_flags &= ~TCP_ECN_OK;
401	}
402
403	static bool tcp_ecn_rcv_ecn_echo(const struct tcp_sock *tp, const struct tcphdr *th)
404	{
405	if (th->ece && !th->syn && (tp->ecn_flags & TCP_ECN_OK))
406	return true;
407	return false;
408	}
409
410	/* Buffer size and advertised window tuning.
411	*
412	* 1. Tuning sk->sk_sndbuf, when connection enters established state.
413	*/
414
415	static void tcp_sndbuf_expand(struct sock *sk)
416	{
417	const struct tcp_sock *tp = tcp_sk(sk);
418	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
419	int sndmem, per_mss;
420	u32 nr_segs;
421
422	/* Worst case is non GSO/TSO : each frame consumes one skb
423	* and skb->head is kmalloced using power of two area of memory
424	*/
425	per_mss = max_t(u32, tp->rx_opt.mss_clamp, tp->mss_cache) +
426	MAX_TCP_HEADER +
427	SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
428
429	per_mss = roundup_pow_of_two(per_mss) +
430	SKB_DATA_ALIGN(sizeof(struct sk_buff));
431
432	nr_segs = max_t(u32, TCP_INIT_CWND, tcp_snd_cwnd(tp));
433	nr_segs = max_t(u32, nr_segs, tp->reordering + 1);
434
435	/* Fast Recovery (RFC 5681 3.2) :
436	* Cubic needs 1.7 factor, rounded to 2 to include
437	* extra cushion (application might react slowly to EPOLLOUT)
438	*/
439	sndmem = ca_ops->sndbuf_expand ? ca_ops->sndbuf_expand(sk) : 2;
440	sndmem *= nr_segs * per_mss;
441
442	if (sk->sk_sndbuf < sndmem)
443	WRITE_ONCE(sk->sk_sndbuf,
444	min(sndmem, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[2])));
445	}
446
447	/* 2. Tuning advertised window (window_clamp, rcv_ssthresh)
448	*
449	* All tcp_full_space() is split to two parts: "network" buffer, allocated
450	* forward and advertised in receiver window (tp->rcv_wnd) and
451	* "application buffer", required to isolate scheduling/application
452	* latencies from network.
453	* window_clamp is maximal advertised window. It can be less than
454	* tcp_full_space(), in this case tcp_full_space() - window_clamp
455	* is reserved for "application" buffer. The less window_clamp is
456	* the smoother our behaviour from viewpoint of network, but the lower
457	* throughput and the higher sensitivity of the connection to losses. 8)
458	*
459	* rcv_ssthresh is more strict window_clamp used at "slow start"
460	* phase to predict further behaviour of this connection.
461	* It is used for two goals:
462	* - to enforce header prediction at sender, even when application
463	* requires some significant "application buffer". It is check #1.
464	* - to prevent pruning of receive queue because of misprediction
465	* of receiver window. Check #2.
466	*
467	* The scheme does not work when sender sends good segments opening
468	* window and then starts to feed us spaghetti. But it should work
469	* in common situations. Otherwise, we have to rely on queue collapsing.
470	*/
471
472	/* Slow part of check#2. */
473	static int __tcp_grow_window(const struct sock *sk, const struct sk_buff *skb,
474	unsigned int skbtruesize)
475	{
476	const struct tcp_sock *tp = tcp_sk(sk);
477	/* Optimize this! */
478	int truesize = tcp_win_from_space(sk, space: skbtruesize) >> 1;
479	int window = tcp_win_from_space(sk, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2])) >> 1;
480
481	while (tp->rcv_ssthresh <= window) {
482	if (truesize <= skb->len)
483	return 2 * inet_csk(sk)->icsk_ack.rcv_mss;
484
485	truesize >>= 1;
486	window >>= 1;
487	}
488	return 0;
489	}
490
491	/* Even if skb appears to have a bad len/truesize ratio, TCP coalescing
492	* can play nice with us, as sk_buff and skb->head might be either
493	* freed or shared with up to MAX_SKB_FRAGS segments.
494	* Only give a boost to drivers using page frag(s) to hold the frame(s),
495	* and if no payload was pulled in skb->head before reaching us.
496	*/
497	static u32 truesize_adjust(bool adjust, const struct sk_buff *skb)
498	{
499	u32 truesize = skb->truesize;
500
501	if (adjust && !skb_headlen(skb)) {
502	truesize -= SKB_TRUESIZE(skb_end_offset(skb));
503	/* paranoid check, some drivers might be buggy */
504	if (unlikely((int)truesize < (int)skb->len))
505	truesize = skb->truesize;
506	}
507	return truesize;
508	}
509
510	static void tcp_grow_window(struct sock *sk, const struct sk_buff *skb,
511	bool adjust)
512	{
513	struct tcp_sock *tp = tcp_sk(sk);
514	int room;
515
516	room = min_t(int, tp->window_clamp, tcp_space(sk)) - tp->rcv_ssthresh;
517
518	if (room <= 0)
519	return;
520
521	/* Check #1 */
522	if (!tcp_under_memory_pressure(sk)) {
523	unsigned int truesize = truesize_adjust(adjust, skb);
524	int incr;
525
526	/* Check #2. Increase window, if skb with such overhead
527	* will fit to rcvbuf in future.
528	*/
529	if (tcp_win_from_space(sk, space: truesize) <= skb->len)
530	incr = 2 * tp->advmss;
531	else
532	incr = __tcp_grow_window(sk, skb, skbtruesize: truesize);
533
534	if (incr) {
535	incr = max_t(int, incr, 2 * skb->len);
536	tp->rcv_ssthresh += min(room, incr);
537	inet_csk(sk)->icsk_ack.quick |= 1;
538	}
539	} else {
540	/* Under pressure:
541	* Adjust rcv_ssthresh according to reserved mem
542	*/
543	tcp_adjust_rcv_ssthresh(sk);
544	}
545	}
546
547	/* 3. Try to fixup all. It is made immediately after connection enters
548	* established state.
549	*/
550	static void tcp_init_buffer_space(struct sock *sk)
551	{
552	int tcp_app_win = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_app_win);
553	struct tcp_sock *tp = tcp_sk(sk);
554	int maxwin;
555
556	if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK))
557	tcp_sndbuf_expand(sk);
558
559	tcp_mstamp_refresh(tp);
560	tp->rcvq_space.time = tp->tcp_mstamp;
561	tp->rcvq_space.seq = tp->copied_seq;
562
563	maxwin = tcp_full_space(sk);
564
565	if (tp->window_clamp >= maxwin) {
566	tp->window_clamp = maxwin;
567
568	if (tcp_app_win && maxwin > 4 * tp->advmss)
569	tp->window_clamp = max(maxwin -
570	(maxwin >> tcp_app_win),
571	4 * tp->advmss);
572	}
573
574	/* Force reservation of one segment. */
575	if (tcp_app_win &&
576	tp->window_clamp > 2 * tp->advmss &&
577	tp->window_clamp + tp->advmss > maxwin)
578	tp->window_clamp = max(2 * tp->advmss, maxwin - tp->advmss);
579
580	tp->rcv_ssthresh = min(tp->rcv_ssthresh, tp->window_clamp);
581	tp->snd_cwnd_stamp = tcp_jiffies32;
582	tp->rcvq_space.space = min3(tp->rcv_ssthresh, tp->rcv_wnd,
583	(u32)TCP_INIT_CWND * tp->advmss);
584	}
585
586	/* 4. Recalculate window clamp after socket hit its memory bounds. */
587	static void tcp_clamp_window(struct sock *sk)
588	{
589	struct tcp_sock *tp = tcp_sk(sk);
590	struct inet_connection_sock *icsk = inet_csk(sk);
591	struct net *net = sock_net(sk);
592	int rmem2;
593
594	icsk->icsk_ack.quick = 0;
595	rmem2 = READ_ONCE(net->ipv4.sysctl_tcp_rmem[2]);
596
597	if (sk->sk_rcvbuf < rmem2 &&
598	!(sk->sk_userlocks & SOCK_RCVBUF_LOCK) &&
599	!tcp_under_memory_pressure(sk) &&
600	sk_memory_allocated(sk) < sk_prot_mem_limits(sk, index: 0)) {
601	WRITE_ONCE(sk->sk_rcvbuf,
602	min(atomic_read(&sk->sk_rmem_alloc), rmem2));
603	}
604	if (atomic_read(v: &sk->sk_rmem_alloc) > sk->sk_rcvbuf)
605	tp->rcv_ssthresh = min(tp->window_clamp, 2U * tp->advmss);
606	}
607
608	/* Initialize RCV_MSS value.
609	* RCV_MSS is an our guess about MSS used by the peer.
610	* We haven't any direct information about the MSS.
611	* It's better to underestimate the RCV_MSS rather than overestimate.
612	* Overestimations make us ACKing less frequently than needed.
613	* Underestimations are more easy to detect and fix by tcp_measure_rcv_mss().
614	*/
615	void tcp_initialize_rcv_mss(struct sock *sk)
616	{
617	const struct tcp_sock *tp = tcp_sk(sk);
618	unsigned int hint = min_t(unsigned int, tp->advmss, tp->mss_cache);
619
620	hint = min(hint, tp->rcv_wnd / 2);
621	hint = min(hint, TCP_MSS_DEFAULT);
622	hint = max(hint, TCP_MIN_MSS);
623
624	inet_csk(sk)->icsk_ack.rcv_mss = hint;
625	}
626	EXPORT_SYMBOL(tcp_initialize_rcv_mss);
627
628	/* Receiver "autotuning" code.
629	*
630	* The algorithm for RTT estimation w/o timestamps is based on
631	* Dynamic Right-Sizing (DRS) by Wu Feng and Mike Fisk of LANL.
632	* <https://public.lanl.gov/radiant/pubs.html#DRS>
633	*
634	* More detail on this code can be found at
635	* <http://staff.psc.edu/jheffner/>,
636	* though this reference is out of date. A new paper
637	* is pending.
638	*/
639	static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep)
640	{
641	u32 new_sample = tp->rcv_rtt_est.rtt_us;
642	long m = sample;
643
644	if (new_sample != 0) {
645	/* If we sample in larger samples in the non-timestamp
646	* case, we could grossly overestimate the RTT especially
647	* with chatty applications or bulk transfer apps which
648	* are stalled on filesystem I/O.
649	*
650	* Also, since we are only going for a minimum in the
651	* non-timestamp case, we do not smooth things out
652	* else with timestamps disabled convergence takes too
653	* long.
654	*/
655	if (!win_dep) {
656	m -= (new_sample >> 3);
657	new_sample += m;
658	} else {
659	m <<= 3;
660	if (m < new_sample)
661	new_sample = m;
662	}
663	} else {
664	/* No previous measure. */
665	new_sample = m << 3;
666	}
667
668	tp->rcv_rtt_est.rtt_us = new_sample;
669	}
670
671	static inline void tcp_rcv_rtt_measure(struct tcp_sock *tp)
672	{
673	u32 delta_us;
674
675	if (tp->rcv_rtt_est.time == 0)
676	goto new_measure;
677	if (before(seq1: tp->rcv_nxt, seq2: tp->rcv_rtt_est.seq))
678	return;
679	delta_us = tcp_stamp_us_delta(t1: tp->tcp_mstamp, t0: tp->rcv_rtt_est.time);
680	if (!delta_us)
681	delta_us = 1;
682	tcp_rcv_rtt_update(tp, sample: delta_us, win_dep: 1);
683
684	new_measure:
685	tp->rcv_rtt_est.seq = tp->rcv_nxt + tp->rcv_wnd;
686	tp->rcv_rtt_est.time = tp->tcp_mstamp;
687	}
688
689	static s32 tcp_rtt_tsopt_us(const struct tcp_sock *tp)
690	{
691	u32 delta, delta_us;
692
693	delta = tcp_time_stamp_ts(tp) - tp->rx_opt.rcv_tsecr;
694	if (tp->tcp_usec_ts)
695	return delta;
696
697	if (likely(delta < INT_MAX / (USEC_PER_SEC / TCP_TS_HZ))) {
698	if (!delta)
699	delta = 1;
700	delta_us = delta * (USEC_PER_SEC / TCP_TS_HZ);
701	return delta_us;
702	}
703	return -1;
704	}
705
706	static inline void tcp_rcv_rtt_measure_ts(struct sock *sk,
707	const struct sk_buff *skb)
708	{
709	struct tcp_sock *tp = tcp_sk(sk);
710
711	if (tp->rx_opt.rcv_tsecr == tp->rcv_rtt_last_tsecr)
712	return;
713	tp->rcv_rtt_last_tsecr = tp->rx_opt.rcv_tsecr;
714
715	if (TCP_SKB_CB(skb)->end_seq -
716	TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss) {
717	s32 delta = tcp_rtt_tsopt_us(tp);
718
719	if (delta >= 0)
720	tcp_rcv_rtt_update(tp, sample: delta, win_dep: 0);
721	}
722	}
723
724	/*
725	* This function should be called every time data is copied to user space.
726	* It calculates the appropriate TCP receive buffer space.
727	*/
728	void tcp_rcv_space_adjust(struct sock *sk)
729	{
730	struct tcp_sock *tp = tcp_sk(sk);
731	u32 copied;
732	int time;
733
734	trace_tcp_rcv_space_adjust(sk);
735
736	tcp_mstamp_refresh(tp);
737	time = tcp_stamp_us_delta(t1: tp->tcp_mstamp, t0: tp->rcvq_space.time);
738	if (time < (tp->rcv_rtt_est.rtt_us >> 3) || tp->rcv_rtt_est.rtt_us == 0)
739	return;
740
741	/* Number of bytes copied to user in last RTT */
742	copied = tp->copied_seq - tp->rcvq_space.seq;
743	if (copied <= tp->rcvq_space.space)
744	goto new_measure;
745
746	/* A bit of theory :
747	* copied = bytes received in previous RTT, our base window
748	* To cope with packet losses, we need a 2x factor
749	* To cope with slow start, and sender growing its cwin by 100 %
750	* every RTT, we need a 4x factor, because the ACK we are sending
751	* now is for the next RTT, not the current one :
752	* <prev RTT . ><current RTT .. ><next RTT .... >
753	*/
754
755	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf) &&
756	!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
757	u64 rcvwin, grow;
758	int rcvbuf;
759
760	/* minimal window to cope with packet losses, assuming
761	* steady state. Add some cushion because of small variations.
762	*/
763	rcvwin = ((u64)copied << 1) + 16 * tp->advmss;
764
765	/* Accommodate for sender rate increase (eg. slow start) */
766	grow = rcvwin * (copied - tp->rcvq_space.space);
767	do_div(grow, tp->rcvq_space.space);
768	rcvwin += (grow << 1);
769
770	rcvbuf = min_t(u64, tcp_space_from_win(sk, rcvwin),
771	READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]));
772	if (rcvbuf > sk->sk_rcvbuf) {
773	WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
774
775	/* Make the window clamp follow along. */
776	tp->window_clamp = tcp_win_from_space(sk, space: rcvbuf);
777	}
778	}
779	tp->rcvq_space.space = copied;
780
781	new_measure:
782	tp->rcvq_space.seq = tp->copied_seq;
783	tp->rcvq_space.time = tp->tcp_mstamp;
784	}
785
786	static void tcp_save_lrcv_flowlabel(struct sock *sk, const struct sk_buff *skb)
787	{
788	#if IS_ENABLED(CONFIG_IPV6)
789	struct inet_connection_sock *icsk = inet_csk(sk);
790
791	if (skb->protocol == htons(ETH_P_IPV6))
792	icsk->icsk_ack.lrcv_flowlabel = ntohl(ip6_flowlabel(ipv6_hdr(skb)));
793	#endif
794	}
795
796	/* There is something which you must keep in mind when you analyze the
797	* behavior of the tp->ato delayed ack timeout interval. When a
798	* connection starts up, we want to ack as quickly as possible. The
799	* problem is that "good" TCP's do slow start at the beginning of data
800	* transmission. The means that until we send the first few ACK's the
801	* sender will sit on his end and only queue most of his data, because
802	* he can only send snd_cwnd unacked packets at any given time. For
803	* each ACK we send, he increments snd_cwnd and transmits more of his
804	* queue. -DaveM
805	*/
806	static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
807	{
808	struct tcp_sock *tp = tcp_sk(sk);
809	struct inet_connection_sock *icsk = inet_csk(sk);
810	u32 now;
811
812	inet_csk_schedule_ack(sk);
813
814	tcp_measure_rcv_mss(sk, skb);
815
816	tcp_rcv_rtt_measure(tp);
817
818	now = tcp_jiffies32;
819
820	if (!icsk->icsk_ack.ato) {
821	/* The _first_ data packet received, initialize
822	* delayed ACK engine.
823	*/
824	tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
825	icsk->icsk_ack.ato = TCP_ATO_MIN;
826	} else {
827	int m = now - icsk->icsk_ack.lrcvtime;
828
829	if (m <= TCP_ATO_MIN / 2) {
830	/* The fastest case is the first. */
831	icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2;
832	} else if (m < icsk->icsk_ack.ato) {
833	icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m;
834	if (icsk->icsk_ack.ato > icsk->icsk_rto)
835	icsk->icsk_ack.ato = icsk->icsk_rto;
836	} else if (m > icsk->icsk_rto) {
837	/* Too long gap. Apparently sender failed to
838	* restart window, so that we send ACKs quickly.
839	*/
840	tcp_incr_quickack(sk, TCP_MAX_QUICKACKS);
841	}
842	}
843	icsk->icsk_ack.lrcvtime = now;
844	tcp_save_lrcv_flowlabel(sk, skb);
845
846	tcp_ecn_check_ce(sk, skb);
847
848	if (skb->len >= 128)
849	tcp_grow_window(sk, skb, adjust: true);
850	}
851
852	/* Called to compute a smoothed rtt estimate. The data fed to this
853	* routine either comes from timestamps, or from segments that were
854	* known _not_ to have been retransmitted [see Karn/Partridge
855	* Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88
856	* piece by Van Jacobson.
857	* NOTE: the next three routines used to be one big routine.
858	* To save cycles in the RFC 1323 implementation it was better to break
859	* it up into three procedures. -- erics
860	*/
861	static void tcp_rtt_estimator(struct sock *sk, long mrtt_us)
862	{
863	struct tcp_sock *tp = tcp_sk(sk);
864	long m = mrtt_us; /* RTT */
865	u32 srtt = tp->srtt_us;
866
867	/* The following amusing code comes from Jacobson's
868	* article in SIGCOMM '88. Note that rtt and mdev
869	* are scaled versions of rtt and mean deviation.
870	* This is designed to be as fast as possible
871	* m stands for "measurement".
872	*
873	* On a 1990 paper the rto value is changed to:
874	* RTO = rtt + 4 * mdev
875	*
876	* Funny. This algorithm seems to be very broken.
877	* These formulae increase RTO, when it should be decreased, increase
878	* too slowly, when it should be increased quickly, decrease too quickly
879	* etc. I guess in BSD RTO takes ONE value, so that it is absolutely
880	* does not matter how to _calculate_ it. Seems, it was trap
881	* that VJ failed to avoid. 8)
882	*/
883	if (srtt != 0) {
884	m -= (srtt >> 3); /* m is now error in rtt est */
885	srtt += m; /* rtt = 7/8 rtt + 1/8 new */
886	if (m < 0) {
887	m = -m; /* m is now abs(error) */
888	m -= (tp->mdev_us >> 2); /* similar update on mdev */
889	/* This is similar to one of Eifel findings.
890	* Eifel blocks mdev updates when rtt decreases.
891	* This solution is a bit different: we use finer gain
892	* for mdev in this case (alpha*beta).
893	* Like Eifel it also prevents growth of rto,
894	* but also it limits too fast rto decreases,
895	* happening in pure Eifel.
896	*/
897	if (m > 0)
898	m >>= 3;
899	} else {
900	m -= (tp->mdev_us >> 2); /* similar update on mdev */
901	}
902	tp->mdev_us += m; /* mdev = 3/4 mdev + 1/4 new */
903	if (tp->mdev_us > tp->mdev_max_us) {
904	tp->mdev_max_us = tp->mdev_us;
905	if (tp->mdev_max_us > tp->rttvar_us)
906	tp->rttvar_us = tp->mdev_max_us;
907	}
908	if (after(tp->snd_una, tp->rtt_seq)) {
909	if (tp->mdev_max_us < tp->rttvar_us)
910	tp->rttvar_us -= (tp->rttvar_us - tp->mdev_max_us) >> 2;
911	tp->rtt_seq = tp->snd_nxt;
912	tp->mdev_max_us = tcp_rto_min_us(sk);
913
914	tcp_bpf_rtt(sk);
915	}
916	} else {
917	/* no previous measure. */
918	srtt = m << 3; /* take the measured time to be rtt */
919	tp->mdev_us = m << 1; /* make sure rto = 3*rtt */
920	tp->rttvar_us = max(tp->mdev_us, tcp_rto_min_us(sk));
921	tp->mdev_max_us = tp->rttvar_us;
922	tp->rtt_seq = tp->snd_nxt;
923
924	tcp_bpf_rtt(sk);
925	}
926	tp->srtt_us = max(1U, srtt);
927	}
928
929	static void tcp_update_pacing_rate(struct sock *sk)
930	{
931	const struct tcp_sock *tp = tcp_sk(sk);
932	u64 rate;
933
934	/* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */
935	rate = (u64)tp->mss_cache * ((USEC_PER_SEC / 100) << 3);
936
937	/* current rate is (cwnd * mss) / srtt
938	* In Slow Start [1], set sk_pacing_rate to 200 % the current rate.
939	* In Congestion Avoidance phase, set it to 120 % the current rate.
940	*
941	* [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh)
942	* If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching
943	* end of slow start and should slow down.
944	*/
945	if (tcp_snd_cwnd(tp) < tp->snd_ssthresh / 2)
946	rate *= READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_pacing_ss_ratio);
947	else
948	rate *= READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_pacing_ca_ratio);
949
950	rate *= max(tcp_snd_cwnd(tp), tp->packets_out);
951
952	if (likely(tp->srtt_us))
953	do_div(rate, tp->srtt_us);
954
955	/* WRITE_ONCE() is needed because sch_fq fetches sk_pacing_rate
956	* without any lock. We want to make sure compiler wont store
957	* intermediate values in this location.
958	*/
959	WRITE_ONCE(sk->sk_pacing_rate,
960	min_t(u64, rate, READ_ONCE(sk->sk_max_pacing_rate)));
961	}
962
963	/* Calculate rto without backoff. This is the second half of Van Jacobson's
964	* routine referred to above.
965	*/
966	static void tcp_set_rto(struct sock *sk)
967	{
968	const struct tcp_sock *tp = tcp_sk(sk);
969	/* Old crap is replaced with new one. 8)
970	*
971	* More seriously:
972	* 1. If rtt variance happened to be less 50msec, it is hallucination.
973	* It cannot be less due to utterly erratic ACK generation made
974	* at least by solaris and freebsd. "Erratic ACKs" has _nothing_
975	* to do with delayed acks, because at cwnd>2 true delack timeout
976	* is invisible. Actually, Linux-2.4 also generates erratic
977	* ACKs in some circumstances.
978	*/
979	inet_csk(sk)->icsk_rto = __tcp_set_rto(tp);
980
981	/* 2. Fixups made earlier cannot be right.
982	* If we do not estimate RTO correctly without them,
983	* all the algo is pure shit and should be replaced
984	* with correct one. It is exactly, which we pretend to do.
985	*/
986
987	/* NOTE: clamping at TCP_RTO_MIN is not required, current algo
988	* guarantees that rto is higher.
989	*/
990	tcp_bound_rto(sk);
991	}
992
993	__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst)
994	{
995	__u32 cwnd = (dst ? dst_metric(dst, RTAX_INITCWND) : 0);
996
997	if (!cwnd)
998	cwnd = TCP_INIT_CWND;
999	return min_t(__u32, cwnd, tp->snd_cwnd_clamp);
1000	}
1001
1002	struct tcp_sacktag_state {
1003	/* Timestamps for earliest and latest never-retransmitted segment
1004	* that was SACKed. RTO needs the earliest RTT to stay conservative,
1005	* but congestion control should still get an accurate delay signal.
1006	*/
1007	u64 first_sackt;
1008	u64 last_sackt;
1009	u32 reord;
1010	u32 sack_delivered;
1011	int flag;
1012	unsigned int mss_now;
1013	struct rate_sample *rate;
1014	};
1015
1016	/* Take a notice that peer is sending D-SACKs. Skip update of data delivery
1017	* and spurious retransmission information if this DSACK is unlikely caused by
1018	* sender's action:
1019	* - DSACKed sequence range is larger than maximum receiver's window.
1020	* - Total no. of DSACKed segments exceed the total no. of retransmitted segs.
1021	*/
1022	static u32 tcp_dsack_seen(struct tcp_sock *tp, u32 start_seq,
1023	u32 end_seq, struct tcp_sacktag_state *state)
1024	{
1025	u32 seq_len, dup_segs = 1;
1026
1027	if (!before(seq1: start_seq, seq2: end_seq))
1028	return 0;
1029
1030	seq_len = end_seq - start_seq;
1031	/* Dubious DSACK: DSACKed range greater than maximum advertised rwnd */
1032	if (seq_len > tp->max_window)
1033	return 0;
1034	if (seq_len > tp->mss_cache)
1035	dup_segs = DIV_ROUND_UP(seq_len, tp->mss_cache);
1036	else if (tp->tlp_high_seq && tp->tlp_high_seq == end_seq)
1037	state->flag |= FLAG_DSACK_TLP;
1038
1039	tp->dsack_dups += dup_segs;
1040	/* Skip the DSACK if dup segs weren't retransmitted by sender */
1041	if (tp->dsack_dups > tp->total_retrans)
1042	return 0;
1043
1044	tp->rx_opt.sack_ok |= TCP_DSACK_SEEN;
1045	/* We increase the RACK ordering window in rounds where we receive
1046	* DSACKs that may have been due to reordering causing RACK to trigger
1047	* a spurious fast recovery. Thus RACK ignores DSACKs that happen
1048	* without having seen reordering, or that match TLP probes (TLP
1049	* is timer-driven, not triggered by RACK).
1050	*/
1051	if (tp->reord_seen && !(state->flag & FLAG_DSACK_TLP))
1052	tp->rack.dsack_seen = 1;
1053
1054	state->flag |= FLAG_DSACKING_ACK;
1055	/* A spurious retransmission is delivered */
1056	state->sack_delivered += dup_segs;
1057
1058	return dup_segs;
1059	}
1060
1061	/* It's reordering when higher sequence was delivered (i.e. sacked) before
1062	* some lower never-retransmitted sequence ("low_seq"). The maximum reordering
1063	* distance is approximated in full-mss packet distance ("reordering").
1064	*/
1065	static void tcp_check_sack_reordering(struct sock *sk, const u32 low_seq,
1066	const int ts)
1067	{
1068	struct tcp_sock *tp = tcp_sk(sk);
1069	const u32 mss = tp->mss_cache;
1070	u32 fack, metric;
1071
1072	fack = tcp_highest_sack_seq(tp);
1073	if (!before(seq1: low_seq, seq2: fack))
1074	return;
1075
1076	metric = fack - low_seq;
1077	if ((metric > tp->reordering * mss) && mss) {
1078	#if FASTRETRANS_DEBUG > 1
1079	pr_debug("Disorder%d %d %u f%u s%u rr%d\n",
1080	tp->rx_opt.sack_ok, inet_csk(sk)->icsk_ca_state,
1081	tp->reordering,
1082	0,
1083	tp->sacked_out,
1084	tp->undo_marker ? tp->undo_retrans : 0);
1085	#endif
1086	tp->reordering = min_t(u32, (metric + mss - 1) / mss,
1087	READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering));
1088	}
1089
1090	/* This exciting event is worth to be remembered. 8) */
1091	tp->reord_seen++;
1092	NET_INC_STATS(sock_net(sk),
1093	ts ? LINUX_MIB_TCPTSREORDER : LINUX_MIB_TCPSACKREORDER);
1094	}
1095
1096	/* This must be called before lost_out or retrans_out are updated
1097	* on a new loss, because we want to know if all skbs previously
1098	* known to be lost have already been retransmitted, indicating
1099	* that this newly lost skb is our next skb to retransmit.
1100	*/
1101	static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb)
1102	{
1103	if ((!tp->retransmit_skb_hint && tp->retrans_out >= tp->lost_out) ||
1104	(tp->retransmit_skb_hint &&
1105	before(TCP_SKB_CB(skb)->seq,
1106	TCP_SKB_CB(tp->retransmit_skb_hint)->seq)))
1107	tp->retransmit_skb_hint = skb;
1108	}
1109
1110	/* Sum the number of packets on the wire we have marked as lost, and
1111	* notify the congestion control module that the given skb was marked lost.
1112	*/
1113	static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb)
1114	{
1115	tp->lost += tcp_skb_pcount(skb);
1116	}
1117
1118	void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb)
1119	{
1120	__u8 sacked = TCP_SKB_CB(skb)->sacked;
1121	struct tcp_sock *tp = tcp_sk(sk);
1122
1123	if (sacked & TCPCB_SACKED_ACKED)
1124	return;
1125
1126	tcp_verify_retransmit_hint(tp, skb);
1127	if (sacked & TCPCB_LOST) {
1128	if (sacked & TCPCB_SACKED_RETRANS) {
1129	/* Account for retransmits that are lost again */
1130	TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_RETRANS;
1131	tp->retrans_out -= tcp_skb_pcount(skb);
1132	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPLOSTRETRANSMIT,
1133	tcp_skb_pcount(skb));
1134	tcp_notify_skb_loss_event(tp, skb);
1135	}
1136	} else {
1137	tp->lost_out += tcp_skb_pcount(skb);
1138	TCP_SKB_CB(skb)->sacked |= TCPCB_LOST;
1139	tcp_notify_skb_loss_event(tp, skb);
1140	}
1141	}
1142
1143	/* Updates the delivered and delivered_ce counts */
1144	static void tcp_count_delivered(struct tcp_sock *tp, u32 delivered,
1145	bool ece_ack)
1146	{
1147	tp->delivered += delivered;
1148	if (ece_ack)
1149	tp->delivered_ce += delivered;
1150	}
1151
1152	/* This procedure tags the retransmission queue when SACKs arrive.
1153	*
1154	* We have three tag bits: SACKED(S), RETRANS(R) and LOST(L).
1155	* Packets in queue with these bits set are counted in variables
1156	* sacked_out, retrans_out and lost_out, correspondingly.
1157	*
1158	* Valid combinations are:
1159	* Tag InFlight Description
1160	* 0 1 - orig segment is in flight.
1161	* S 0 - nothing flies, orig reached receiver.
1162	* L 0 - nothing flies, orig lost by net.
1163	* R 2 - both orig and retransmit are in flight.
1164	* L|R 1 - orig is lost, retransmit is in flight.
1165	* S|R 1 - orig reached receiver, retrans is still in flight.
1166	* (L|S|R is logically valid, it could occur when L|R is sacked,
1167	* but it is equivalent to plain S and code short-circuits it to S.
1168	* L|S is logically invalid, it would mean -1 packet in flight 8))
1169	*
1170	* These 6 states form finite state machine, controlled by the following events:
1171	* 1. New ACK (+SACK) arrives. (tcp_sacktag_write_queue())
1172	* 2. Retransmission. (tcp_retransmit_skb(), tcp_xmit_retransmit_queue())
1173	* 3. Loss detection event of two flavors:
1174	* A. Scoreboard estimator decided the packet is lost.
1175	* A'. Reno "three dupacks" marks head of queue lost.
1176	* B. SACK arrives sacking SND.NXT at the moment, when the
1177	* segment was retransmitted.
1178	* 4. D-SACK added new rule: D-SACK changes any tag to S.
1179	*
1180	* It is pleasant to note, that state diagram turns out to be commutative,
1181	* so that we are allowed not to be bothered by order of our actions,
1182	* when multiple events arrive simultaneously. (see the function below).
1183	*
1184	* Reordering detection.
1185	* --------------------
1186	* Reordering metric is maximal distance, which a packet can be displaced
1187	* in packet stream. With SACKs we can estimate it:
1188	*
1189	* 1. SACK fills old hole and the corresponding segment was not
1190	* ever retransmitted -> reordering. Alas, we cannot use it
1191	* when segment was retransmitted.
1192	* 2. The last flaw is solved with D-SACK. D-SACK arrives
1193	* for retransmitted and already SACKed segment -> reordering..
1194	* Both of these heuristics are not used in Loss state, when we cannot
1195	* account for retransmits accurately.
1196	*
1197	* SACK block validation.
1198	* ----------------------
1199	*
1200	* SACK block range validation checks that the received SACK block fits to
1201	* the expected sequence limits, i.e., it is between SND.UNA and SND.NXT.
1202	* Note that SND.UNA is not included to the range though being valid because
1203	* it means that the receiver is rather inconsistent with itself reporting
1204	* SACK reneging when it should advance SND.UNA. Such SACK block this is
1205	* perfectly valid, however, in light of RFC2018 which explicitly states
1206	* that "SACK block MUST reflect the newest segment. Even if the newest
1207	* segment is going to be discarded ...", not that it looks very clever
1208	* in case of head skb. Due to potentional receiver driven attacks, we
1209	* choose to avoid immediate execution of a walk in write queue due to
1210	* reneging and defer head skb's loss recovery to standard loss recovery
1211	* procedure that will eventually trigger (nothing forbids us doing this).
1212	*
1213	* Implements also blockage to start_seq wrap-around. Problem lies in the
1214	* fact that though start_seq (s) is before end_seq (i.e., not reversed),
1215	* there's no guarantee that it will be before snd_nxt (n). The problem
1216	* happens when start_seq resides between end_seq wrap (e_w) and snd_nxt
1217	* wrap (s_w):
1218	*
1219	* <- outs wnd -> <- wrapzone ->
1220	* u e n u_w e_w s n_w
1221	* | | | | | | |
1222	* |<------------+------+----- TCP seqno space --------------+---------->|
1223	* ...-- <2^31 ->| |<--------...
1224	* ...---- >2^31 ------>| |<--------...
1225	*
1226	* Current code wouldn't be vulnerable but it's better still to discard such
1227	* crazy SACK blocks. Doing this check for start_seq alone closes somewhat
1228	* similar case (end_seq after snd_nxt wrap) as earlier reversed check in
1229	* snd_nxt wrap -> snd_una region will then become "well defined", i.e.,
1230	* equal to the ideal case (infinite seqno space without wrap caused issues).
1231	*
1232	* With D-SACK the lower bound is extended to cover sequence space below
1233	* SND.UNA down to undo_marker, which is the last point of interest. Yet
1234	* again, D-SACK block must not to go across snd_una (for the same reason as
1235	* for the normal SACK blocks, explained above). But there all simplicity
1236	* ends, TCP might receive valid D-SACKs below that. As long as they reside
1237	* fully below undo_marker they do not affect behavior in anyway and can
1238	* therefore be safely ignored. In rare cases (which are more or less
1239	* theoretical ones), the D-SACK will nicely cross that boundary due to skb
1240	* fragmentation and packet reordering past skb's retransmission. To consider
1241	* them correctly, the acceptable range must be extended even more though
1242	* the exact amount is rather hard to quantify. However, tp->max_window can
1243	* be used as an exaggerated estimate.
1244	*/
1245	static bool tcp_is_sackblock_valid(struct tcp_sock *tp, bool is_dsack,
1246	u32 start_seq, u32 end_seq)
1247	{
1248	/* Too far in future, or reversed (interpretation is ambiguous) */
1249	if (after(end_seq, tp->snd_nxt) || !before(seq1: start_seq, seq2: end_seq))
1250	return false;
1251
1252	/* Nasty start_seq wrap-around check (see comments above) */
1253	if (!before(seq1: start_seq, seq2: tp->snd_nxt))
1254	return false;
1255
1256	/* In outstanding window? ...This is valid exit for D-SACKs too.
1257	* start_seq == snd_una is non-sensical (see comments above)
1258	*/
1259	if (after(start_seq, tp->snd_una))
1260	return true;
1261
1262	if (!is_dsack || !tp->undo_marker)
1263	return false;
1264
1265	/* ...Then it's D-SACK, and must reside below snd_una completely */
1266	if (after(end_seq, tp->snd_una))
1267	return false;
1268
1269	if (!before(seq1: start_seq, seq2: tp->undo_marker))
1270	return true;
1271
1272	/* Too old */
1273	if (!after(end_seq, tp->undo_marker))
1274	return false;
1275
1276	/* Undo_marker boundary crossing (overestimates a lot). Known already:
1277	* start_seq < undo_marker and end_seq >= undo_marker.
1278	*/
1279	return !before(seq1: start_seq, seq2: end_seq - tp->max_window);
1280	}
1281
1282	static bool tcp_check_dsack(struct sock *sk, const struct sk_buff *ack_skb,
1283	struct tcp_sack_block_wire *sp, int num_sacks,
1284	u32 prior_snd_una, struct tcp_sacktag_state *state)
1285	{
1286	struct tcp_sock *tp = tcp_sk(sk);
1287	u32 start_seq_0 = get_unaligned_be32(p: &sp[0].start_seq);
1288	u32 end_seq_0 = get_unaligned_be32(p: &sp[0].end_seq);
1289	u32 dup_segs;
1290
1291	if (before(seq1: start_seq_0, TCP_SKB_CB(ack_skb)->ack_seq)) {
1292	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECV);
1293	} else if (num_sacks > 1) {
1294	u32 end_seq_1 = get_unaligned_be32(p: &sp[1].end_seq);
1295	u32 start_seq_1 = get_unaligned_be32(p: &sp[1].start_seq);
1296
1297	if (after(end_seq_0, end_seq_1) || before(seq1: start_seq_0, seq2: start_seq_1))
1298	return false;
1299	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKOFORECV);
1300	} else {
1301	return false;
1302	}
1303
1304	dup_segs = tcp_dsack_seen(tp, start_seq: start_seq_0, end_seq: end_seq_0, state);
1305	if (!dup_segs) { /* Skip dubious DSACK */
1306	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKIGNOREDDUBIOUS);
1307	return false;
1308	}
1309
1310	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECVSEGS, dup_segs);
1311
1312	/* D-SACK for already forgotten data... Do dumb counting. */
1313	if (tp->undo_marker && tp->undo_retrans > 0 &&
1314	!after(end_seq_0, prior_snd_una) &&
1315	after(end_seq_0, tp->undo_marker))
1316	tp->undo_retrans = max_t(int, 0, tp->undo_retrans - dup_segs);
1317
1318	return true;
1319	}
1320
1321	/* Check if skb is fully within the SACK block. In presence of GSO skbs,
1322	* the incoming SACK may not exactly match but we can find smaller MSS
1323	* aligned portion of it that matches. Therefore we might need to fragment
1324	* which may fail and creates some hassle (caller must handle error case
1325	* returns).
1326	*
1327	* FIXME: this could be merged to shift decision code
1328	*/
1329	static int tcp_match_skb_to_sack(struct sock *sk, struct sk_buff *skb,
1330	u32 start_seq, u32 end_seq)
1331	{
1332	int err;
1333	bool in_sack;
1334	unsigned int pkt_len;
1335	unsigned int mss;
1336
1337	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
1338	!before(seq1: end_seq, TCP_SKB_CB(skb)->end_seq);
1339
1340	if (tcp_skb_pcount(skb) > 1 && !in_sack &&
1341	after(TCP_SKB_CB(skb)->end_seq, start_seq)) {
1342	mss = tcp_skb_mss(skb);
1343	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
1344
1345	if (!in_sack) {
1346	pkt_len = start_seq - TCP_SKB_CB(skb)->seq;
1347	if (pkt_len < mss)
1348	pkt_len = mss;
1349	} else {
1350	pkt_len = end_seq - TCP_SKB_CB(skb)->seq;
1351	if (pkt_len < mss)
1352	return -EINVAL;
1353	}
1354
1355	/* Round if necessary so that SACKs cover only full MSSes
1356	* and/or the remaining small portion (if present)
1357	*/
1358	if (pkt_len > mss) {
1359	unsigned int new_len = (pkt_len / mss) * mss;
1360	if (!in_sack && new_len < pkt_len)
1361	new_len += mss;
1362	pkt_len = new_len;
1363	}
1364
1365	if (pkt_len >= skb->len && !in_sack)
1366	return 0;
1367
1368	err = tcp_fragment(sk, tcp_queue: TCP_FRAG_IN_RTX_QUEUE, skb,
1369	len: pkt_len, mss_now: mss, GFP_ATOMIC);
1370	if (err < 0)
1371	return err;
1372	}
1373
1374	return in_sack;
1375	}
1376
1377	/* Mark the given newly-SACKed range as such, adjusting counters and hints. */
1378	static u8 tcp_sacktag_one(struct sock *sk,
1379	struct tcp_sacktag_state *state, u8 sacked,
1380	u32 start_seq, u32 end_seq,
1381	int dup_sack, int pcount,
1382	u64 xmit_time)
1383	{
1384	struct tcp_sock *tp = tcp_sk(sk);
1385
1386	/* Account D-SACK for retransmitted packet. */
1387	if (dup_sack && (sacked & TCPCB_RETRANS)) {
1388	if (tp->undo_marker && tp->undo_retrans > 0 &&
1389	after(end_seq, tp->undo_marker))
1390	tp->undo_retrans = max_t(int, 0, tp->undo_retrans - pcount);
1391	if ((sacked & TCPCB_SACKED_ACKED) &&
1392	before(seq1: start_seq, seq2: state->reord))
1393	state->reord = start_seq;
1394	}
1395
1396	/* Nothing to do; acked frame is about to be dropped (was ACKed). */
1397	if (!after(end_seq, tp->snd_una))
1398	return sacked;
1399
1400	if (!(sacked & TCPCB_SACKED_ACKED)) {
1401	tcp_rack_advance(tp, sacked, end_seq, xmit_time);
1402
1403	if (sacked & TCPCB_SACKED_RETRANS) {
1404	/* If the segment is not tagged as lost,
1405	* we do not clear RETRANS, believing
1406	* that retransmission is still in flight.
1407	*/
1408	if (sacked & TCPCB_LOST) {
1409	sacked &= ~(TCPCB_LOST|TCPCB_SACKED_RETRANS);
1410	tp->lost_out -= pcount;
1411	tp->retrans_out -= pcount;
1412	}
1413	} else {
1414	if (!(sacked & TCPCB_RETRANS)) {
1415	/* New sack for not retransmitted frame,
1416	* which was in hole. It is reordering.
1417	*/
1418	if (before(seq1: start_seq,
1419	seq2: tcp_highest_sack_seq(tp)) &&
1420	before(seq1: start_seq, seq2: state->reord))
1421	state->reord = start_seq;
1422
1423	if (!after(end_seq, tp->high_seq))
1424	state->flag |= FLAG_ORIG_SACK_ACKED;
1425	if (state->first_sackt == 0)
1426	state->first_sackt = xmit_time;
1427	state->last_sackt = xmit_time;
1428	}
1429
1430	if (sacked & TCPCB_LOST) {
1431	sacked &= ~TCPCB_LOST;
1432	tp->lost_out -= pcount;
1433	}
1434	}
1435
1436	sacked |= TCPCB_SACKED_ACKED;
1437	state->flag |= FLAG_DATA_SACKED;
1438	tp->sacked_out += pcount;
1439	/* Out-of-order packets delivered */
1440	state->sack_delivered += pcount;
1441
1442	/* Lost marker hint past SACKed? Tweak RFC3517 cnt */
1443	if (tp->lost_skb_hint &&
1444	before(seq1: start_seq, TCP_SKB_CB(tp->lost_skb_hint)->seq))
1445	tp->lost_cnt_hint += pcount;
1446	}
1447
1448	/* D-SACK. We can detect redundant retransmission in S|R and plain R
1449	* frames and clear it. undo_retrans is decreased above, L|R frames
1450	* are accounted above as well.
1451	*/
1452	if (dup_sack && (sacked & TCPCB_SACKED_RETRANS)) {
1453	sacked &= ~TCPCB_SACKED_RETRANS;
1454	tp->retrans_out -= pcount;
1455	}
1456
1457	return sacked;
1458	}
1459
1460	/* Shift newly-SACKed bytes from this skb to the immediately previous
1461	* already-SACKed sk_buff. Mark the newly-SACKed bytes as such.
1462	*/
1463	static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev,
1464	struct sk_buff *skb,
1465	struct tcp_sacktag_state *state,
1466	unsigned int pcount, int shifted, int mss,
1467	bool dup_sack)
1468	{
1469	struct tcp_sock *tp = tcp_sk(sk);
1470	u32 start_seq = TCP_SKB_CB(skb)->seq; /* start of newly-SACKed */
1471	u32 end_seq = start_seq + shifted; /* end of newly-SACKed */
1472
1473	BUG_ON(!pcount);
1474
1475	/* Adjust counters and hints for the newly sacked sequence
1476	* range but discard the return value since prev is already
1477	* marked. We must tag the range first because the seq
1478	* advancement below implicitly advances
1479	* tcp_highest_sack_seq() when skb is highest_sack.
1480	*/
1481	tcp_sacktag_one(sk, state, TCP_SKB_CB(skb)->sacked,
1482	start_seq, end_seq, dup_sack, pcount,
1483	xmit_time: tcp_skb_timestamp_us(skb));
1484	tcp_rate_skb_delivered(sk, skb, rs: state->rate);
1485
1486	if (skb == tp->lost_skb_hint)
1487	tp->lost_cnt_hint += pcount;
1488
1489	TCP_SKB_CB(prev)->end_seq += shifted;
1490	TCP_SKB_CB(skb)->seq += shifted;
1491
1492	tcp_skb_pcount_add(skb: prev, segs: pcount);
1493	WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount);
1494	tcp_skb_pcount_add(skb, segs: -pcount);
1495
1496	/* When we're adding to gso_segs == 1, gso_size will be zero,
1497	* in theory this shouldn't be necessary but as long as DSACK
1498	* code can come after this skb later on it's better to keep
1499	* setting gso_size to something.
1500	*/
1501	if (!TCP_SKB_CB(prev)->tcp_gso_size)
1502	TCP_SKB_CB(prev)->tcp_gso_size = mss;
1503
1504	/* CHECKME: To clear or not to clear? Mimics normal skb currently */
1505	if (tcp_skb_pcount(skb) <= 1)
1506	TCP_SKB_CB(skb)->tcp_gso_size = 0;
1507
1508	/* Difference in this won't matter, both ACKed by the same cumul. ACK */
1509	TCP_SKB_CB(prev)->sacked |= (TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS);
1510
1511	if (skb->len > 0) {
1512	BUG_ON(!tcp_skb_pcount(skb));
1513	NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTED);
1514	return false;
1515	}
1516
1517	/* Whole SKB was eaten :-) */
1518
1519	if (skb == tp->retransmit_skb_hint)
1520	tp->retransmit_skb_hint = prev;
1521	if (skb == tp->lost_skb_hint) {
1522	tp->lost_skb_hint = prev;
1523	tp->lost_cnt_hint -= tcp_skb_pcount(skb: prev);
1524	}
1525
1526	TCP_SKB_CB(prev)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1527	TCP_SKB_CB(prev)->eor = TCP_SKB_CB(skb)->eor;
1528	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1529	TCP_SKB_CB(prev)->end_seq++;
1530
1531	if (skb == tcp_highest_sack(sk))
1532	tcp_advance_highest_sack(sk, skb);
1533
1534	tcp_skb_collapse_tstamp(skb: prev, next_skb: skb);
1535	if (unlikely(TCP_SKB_CB(prev)->tx.delivered_mstamp))
1536	TCP_SKB_CB(prev)->tx.delivered_mstamp = 0;
1537
1538	tcp_rtx_queue_unlink_and_free(skb, sk);
1539
1540	NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKMERGED);
1541
1542	return true;
1543	}
1544
1545	/* I wish gso_size would have a bit more sane initialization than
1546	* something-or-zero which complicates things
1547	*/
1548	static int tcp_skb_seglen(const struct sk_buff *skb)
1549	{
1550	return tcp_skb_pcount(skb) == 1 ? skb->len : tcp_skb_mss(skb);
1551	}
1552
1553	/* Shifting pages past head area doesn't work */
1554	static int skb_can_shift(const struct sk_buff *skb)
1555	{
1556	return !skb_headlen(skb) && skb_is_nonlinear(skb);
1557	}
1558
1559	int tcp_skb_shift(struct sk_buff *to, struct sk_buff *from,
1560	int pcount, int shiftlen)
1561	{
1562	/* TCP min gso_size is 8 bytes (TCP_MIN_GSO_SIZE)
1563	* Since TCP_SKB_CB(skb)->tcp_gso_segs is 16 bits, we need
1564	* to make sure not storing more than 65535 * 8 bytes per skb,
1565	* even if current MSS is bigger.
1566	*/
1567	if (unlikely(to->len + shiftlen >= 65535 * TCP_MIN_GSO_SIZE))
1568	return 0;
1569	if (unlikely(tcp_skb_pcount(to) + pcount > 65535))
1570	return 0;
1571	return skb_shift(tgt: to, skb: from, shiftlen);
1572	}
1573
1574	/* Try collapsing SACK blocks spanning across multiple skbs to a single
1575	* skb.
1576	*/
1577	static struct sk_buff *tcp_shift_skb_data(struct sock *sk, struct sk_buff *skb,
1578	struct tcp_sacktag_state *state,
1579	u32 start_seq, u32 end_seq,
1580	bool dup_sack)
1581	{
1582	struct tcp_sock *tp = tcp_sk(sk);
1583	struct sk_buff *prev;
1584	int mss;
1585	int pcount = 0;
1586	int len;
1587	int in_sack;
1588
1589	/* Normally R but no L won't result in plain S */
1590	if (!dup_sack &&
1591	(TCP_SKB_CB(skb)->sacked & (TCPCB_LOST|TCPCB_SACKED_RETRANS)) == TCPCB_SACKED_RETRANS)
1592	goto fallback;
1593	if (!skb_can_shift(skb))
1594	goto fallback;
1595	/* This frame is about to be dropped (was ACKed). */
1596	if (!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
1597	goto fallback;
1598
1599	/* Can only happen with delayed DSACK + discard craziness */
1600	prev = skb_rb_prev(skb);
1601	if (!prev)
1602	goto fallback;
1603
1604	if ((TCP_SKB_CB(prev)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED)
1605	goto fallback;
1606
1607	if (!tcp_skb_can_collapse(to: prev, from: skb))
1608	goto fallback;
1609
1610	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq) &&
1611	!before(seq1: end_seq, TCP_SKB_CB(skb)->end_seq);
1612
1613	if (in_sack) {
1614	len = skb->len;
1615	pcount = tcp_skb_pcount(skb);
1616	mss = tcp_skb_seglen(skb);
1617
1618	/* TODO: Fix DSACKs to not fragment already SACKed and we can
1619	* drop this restriction as unnecessary
1620	*/
1621	if (mss != tcp_skb_seglen(skb: prev))
1622	goto fallback;
1623	} else {
1624	if (!after(TCP_SKB_CB(skb)->end_seq, start_seq))
1625	goto noop;
1626	/* CHECKME: This is non-MSS split case only?, this will
1627	* cause skipped skbs due to advancing loop btw, original
1628	* has that feature too
1629	*/
1630	if (tcp_skb_pcount(skb) <= 1)
1631	goto noop;
1632
1633	in_sack = !after(start_seq, TCP_SKB_CB(skb)->seq);
1634	if (!in_sack) {
1635	/* TODO: head merge to next could be attempted here
1636	* if (!after(TCP_SKB_CB(skb)->end_seq, end_seq)),
1637	* though it might not be worth of the additional hassle
1638	*
1639	* ...we can probably just fallback to what was done
1640	* previously. We could try merging non-SACKed ones
1641	* as well but it probably isn't going to buy off
1642	* because later SACKs might again split them, and
1643	* it would make skb timestamp tracking considerably
1644	* harder problem.
1645	*/
1646	goto fallback;
1647	}
1648
1649	len = end_seq - TCP_SKB_CB(skb)->seq;
1650	BUG_ON(len < 0);
1651	BUG_ON(len > skb->len);
1652
1653	/* MSS boundaries should be honoured or else pcount will
1654	* severely break even though it makes things bit trickier.
1655	* Optimize common case to avoid most of the divides
1656	*/
1657	mss = tcp_skb_mss(skb);
1658
1659	/* TODO: Fix DSACKs to not fragment already SACKed and we can
1660	* drop this restriction as unnecessary
1661	*/
1662	if (mss != tcp_skb_seglen(skb: prev))
1663	goto fallback;
1664
1665	if (len == mss) {
1666	pcount = 1;
1667	} else if (len < mss) {
1668	goto noop;
1669	} else {
1670	pcount = len / mss;
1671	len = pcount * mss;
1672	}
1673	}
1674
1675	/* tcp_sacktag_one() won't SACK-tag ranges below snd_una */
1676	if (!after(TCP_SKB_CB(skb)->seq + len, tp->snd_una))
1677	goto fallback;
1678
1679	if (!tcp_skb_shift(to: prev, from: skb, pcount, shiftlen: len))
1680	goto fallback;
1681	if (!tcp_shifted_skb(sk, prev, skb, state, pcount, shifted: len, mss, dup_sack))
1682	goto out;
1683
1684	/* Hole filled allows collapsing with the next as well, this is very
1685	* useful when hole on every nth skb pattern happens
1686	*/
1687	skb = skb_rb_next(prev);
1688	if (!skb)
1689	goto out;
1690
1691	if (!skb_can_shift(skb) ||
1692	((TCP_SKB_CB(skb)->sacked & TCPCB_TAGBITS) != TCPCB_SACKED_ACKED) ||
1693	(mss != tcp_skb_seglen(skb)))
1694	goto out;
1695
1696	if (!tcp_skb_can_collapse(to: prev, from: skb))
1697	goto out;
1698	len = skb->len;
1699	pcount = tcp_skb_pcount(skb);
1700	if (tcp_skb_shift(to: prev, from: skb, pcount, shiftlen: len))
1701	tcp_shifted_skb(sk, prev, skb, state, pcount,
1702	shifted: len, mss, dup_sack: 0);
1703
1704	out:
1705	return prev;
1706
1707	noop:
1708	return skb;
1709
1710	fallback:
1711	NET_INC_STATS(sock_net(sk), LINUX_MIB_SACKSHIFTFALLBACK);
1712	return NULL;
1713	}
1714
1715	static struct sk_buff *tcp_sacktag_walk(struct sk_buff *skb, struct sock *sk,
1716	struct tcp_sack_block *next_dup,
1717	struct tcp_sacktag_state *state,
1718	u32 start_seq, u32 end_seq,
1719	bool dup_sack_in)
1720	{
1721	struct tcp_sock *tp = tcp_sk(sk);
1722	struct sk_buff *tmp;
1723
1724	skb_rbtree_walk_from(skb) {
1725	int in_sack = 0;
1726	bool dup_sack = dup_sack_in;
1727
1728	/* queue is in-order => we can short-circuit the walk early */
1729	if (!before(TCP_SKB_CB(skb)->seq, seq2: end_seq))
1730	break;
1731
1732	if (next_dup &&
1733	before(TCP_SKB_CB(skb)->seq, seq2: next_dup->end_seq)) {
1734	in_sack = tcp_match_skb_to_sack(sk, skb,
1735	start_seq: next_dup->start_seq,
1736	end_seq: next_dup->end_seq);
1737	if (in_sack > 0)
1738	dup_sack = true;
1739	}
1740
1741	/* skb reference here is a bit tricky to get right, since
1742	* shifting can eat and free both this skb and the next,
1743	* so not even _safe variant of the loop is enough.
1744	*/
1745	if (in_sack <= 0) {
1746	tmp = tcp_shift_skb_data(sk, skb, state,
1747	start_seq, end_seq, dup_sack);
1748	if (tmp) {
1749	if (tmp != skb) {
1750	skb = tmp;
1751	continue;
1752	}
1753
1754	in_sack = 0;
1755	} else {
1756	in_sack = tcp_match_skb_to_sack(sk, skb,
1757	start_seq,
1758	end_seq);
1759	}
1760	}
1761
1762	if (unlikely(in_sack < 0))
1763	break;
1764
1765	if (in_sack) {
1766	TCP_SKB_CB(skb)->sacked =
1767	tcp_sacktag_one(sk,
1768	state,
1769	TCP_SKB_CB(skb)->sacked,
1770	TCP_SKB_CB(skb)->seq,
1771	TCP_SKB_CB(skb)->end_seq,
1772	dup_sack,
1773	pcount: tcp_skb_pcount(skb),
1774	xmit_time: tcp_skb_timestamp_us(skb));
1775	tcp_rate_skb_delivered(sk, skb, rs: state->rate);
1776	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1777	list_del_init(entry: &skb->tcp_tsorted_anchor);
1778
1779	if (!before(TCP_SKB_CB(skb)->seq,
1780	seq2: tcp_highest_sack_seq(tp)))
1781	tcp_advance_highest_sack(sk, skb);
1782	}
1783	}
1784	return skb;
1785	}
1786
1787	static struct sk_buff *tcp_sacktag_bsearch(struct sock *sk, u32 seq)
1788	{
1789	struct rb_node *parent, **p = &sk->tcp_rtx_queue.rb_node;
1790	struct sk_buff *skb;
1791
1792	while (*p) {
1793	parent = *p;
1794	skb = rb_to_skb(parent);
1795	if (before(seq1: seq, TCP_SKB_CB(skb)->seq)) {
1796	p = &parent->rb_left;
1797	continue;
1798	}
1799	if (!before(seq1: seq, TCP_SKB_CB(skb)->end_seq)) {
1800	p = &parent->rb_right;
1801	continue;
1802	}
1803	return skb;
1804	}
1805	return NULL;
1806	}
1807
1808	static struct sk_buff *tcp_sacktag_skip(struct sk_buff *skb, struct sock *sk,
1809	u32 skip_to_seq)
1810	{
1811	if (skb && after(TCP_SKB_CB(skb)->seq, skip_to_seq))
1812	return skb;
1813
1814	return tcp_sacktag_bsearch(sk, seq: skip_to_seq);
1815	}
1816
1817	static struct sk_buff *tcp_maybe_skipping_dsack(struct sk_buff *skb,
1818	struct sock *sk,
1819	struct tcp_sack_block *next_dup,
1820	struct tcp_sacktag_state *state,
1821	u32 skip_to_seq)
1822	{
1823	if (!next_dup)
1824	return skb;
1825
1826	if (before(seq1: next_dup->start_seq, seq2: skip_to_seq)) {
1827	skb = tcp_sacktag_skip(skb, sk, skip_to_seq: next_dup->start_seq);
1828	skb = tcp_sacktag_walk(skb, sk, NULL, state,
1829	start_seq: next_dup->start_seq, end_seq: next_dup->end_seq,
1830	dup_sack_in: 1);
1831	}
1832
1833	return skb;
1834	}
1835
1836	static int tcp_sack_cache_ok(const struct tcp_sock *tp, const struct tcp_sack_block *cache)
1837	{
1838	return cache < tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
1839	}
1840
1841	static int
1842	tcp_sacktag_write_queue(struct sock *sk, const struct sk_buff *ack_skb,
1843	u32 prior_snd_una, struct tcp_sacktag_state *state)
1844	{
1845	struct tcp_sock *tp = tcp_sk(sk);
1846	const unsigned char *ptr = (skb_transport_header(skb: ack_skb) +
1847	TCP_SKB_CB(ack_skb)->sacked);
1848	struct tcp_sack_block_wire *sp_wire = (struct tcp_sack_block_wire *)(ptr+2);
1849	struct tcp_sack_block sp[TCP_NUM_SACKS];
1850	struct tcp_sack_block *cache;
1851	struct sk_buff *skb;
1852	int num_sacks = min(TCP_NUM_SACKS, (ptr[1] - TCPOLEN_SACK_BASE) >> 3);
1853	int used_sacks;
1854	bool found_dup_sack = false;
1855	int i, j;
1856	int first_sack_index;
1857
1858	state->flag = 0;
1859	state->reord = tp->snd_nxt;
1860
1861	if (!tp->sacked_out)
1862	tcp_highest_sack_reset(sk);
1863
1864	found_dup_sack = tcp_check_dsack(sk, ack_skb, sp: sp_wire,
1865	num_sacks, prior_snd_una, state);
1866
1867	/* Eliminate too old ACKs, but take into
1868	* account more or less fresh ones, they can
1869	* contain valid SACK info.
1870	*/
1871	if (before(TCP_SKB_CB(ack_skb)->ack_seq, seq2: prior_snd_una - tp->max_window))
1872	return 0;
1873
1874	if (!tp->packets_out)
1875	goto out;
1876
1877	used_sacks = 0;
1878	first_sack_index = 0;
1879	for (i = 0; i < num_sacks; i++) {
1880	bool dup_sack = !i && found_dup_sack;
1881
1882	sp[used_sacks].start_seq = get_unaligned_be32(p: &sp_wire[i].start_seq);
1883	sp[used_sacks].end_seq = get_unaligned_be32(p: &sp_wire[i].end_seq);
1884
1885	if (!tcp_is_sackblock_valid(tp, is_dsack: dup_sack,
1886	start_seq: sp[used_sacks].start_seq,
1887	end_seq: sp[used_sacks].end_seq)) {
1888	int mib_idx;
1889
1890	if (dup_sack) {
1891	if (!tp->undo_marker)
1892	mib_idx = LINUX_MIB_TCPDSACKIGNOREDNOUNDO;
1893	else
1894	mib_idx = LINUX_MIB_TCPDSACKIGNOREDOLD;
1895	} else {
1896	/* Don't count olds caused by ACK reordering */
1897	if ((TCP_SKB_CB(ack_skb)->ack_seq != tp->snd_una) &&
1898	!after(sp[used_sacks].end_seq, tp->snd_una))
1899	continue;
1900	mib_idx = LINUX_MIB_TCPSACKDISCARD;
1901	}
1902
1903	NET_INC_STATS(sock_net(sk), mib_idx);
1904	if (i == 0)
1905	first_sack_index = -1;
1906	continue;
1907	}
1908
1909	/* Ignore very old stuff early */
1910	if (!after(sp[used_sacks].end_seq, prior_snd_una)) {
1911	if (i == 0)
1912	first_sack_index = -1;
1913	continue;
1914	}
1915
1916	used_sacks++;
1917	}
1918
1919	/* order SACK blocks to allow in order walk of the retrans queue */
1920	for (i = used_sacks - 1; i > 0; i--) {
1921	for (j = 0; j < i; j++) {
1922	if (after(sp[j].start_seq, sp[j + 1].start_seq)) {
1923	swap(sp[j], sp[j + 1]);
1924
1925	/* Track where the first SACK block goes to */
1926	if (j == first_sack_index)
1927	first_sack_index = j + 1;
1928	}
1929	}
1930	}
1931
1932	state->mss_now = tcp_current_mss(sk);
1933	skb = NULL;
1934	i = 0;
1935
1936	if (!tp->sacked_out) {
1937	/* It's already past, so skip checking against it */
1938	cache = tp->recv_sack_cache + ARRAY_SIZE(tp->recv_sack_cache);
1939	} else {
1940	cache = tp->recv_sack_cache;
1941	/* Skip empty blocks in at head of the cache */
1942	while (tcp_sack_cache_ok(tp, cache) && !cache->start_seq &&
1943	!cache->end_seq)
1944	cache++;
1945	}
1946
1947	while (i < used_sacks) {
1948	u32 start_seq = sp[i].start_seq;
1949	u32 end_seq = sp[i].end_seq;
1950	bool dup_sack = (found_dup_sack && (i == first_sack_index));
1951	struct tcp_sack_block *next_dup = NULL;
1952
1953	if (found_dup_sack && ((i + 1) == first_sack_index))
1954	next_dup = &sp[i + 1];
1955
1956	/* Skip too early cached blocks */
1957	while (tcp_sack_cache_ok(tp, cache) &&
1958	!before(seq1: start_seq, seq2: cache->end_seq))
1959	cache++;
1960
1961	/* Can skip some work by looking recv_sack_cache? */
1962	if (tcp_sack_cache_ok(tp, cache) && !dup_sack &&
1963	after(end_seq, cache->start_seq)) {
1964
1965	/* Head todo? */
1966	if (before(seq1: start_seq, seq2: cache->start_seq)) {
1967	skb = tcp_sacktag_skip(skb, sk, skip_to_seq: start_seq);
1968	skb = tcp_sacktag_walk(skb, sk, next_dup,
1969	state,
1970	start_seq,
1971	end_seq: cache->start_seq,
1972	dup_sack_in: dup_sack);
1973	}
1974
1975	/* Rest of the block already fully processed? */
1976	if (!after(end_seq, cache->end_seq))
1977	goto advance_sp;
1978
1979	skb = tcp_maybe_skipping_dsack(skb, sk, next_dup,
1980	state,
1981	skip_to_seq: cache->end_seq);
1982
1983	/* ...tail remains todo... */
1984	if (tcp_highest_sack_seq(tp) == cache->end_seq) {
1985	/* ...but better entrypoint exists! */
1986	skb = tcp_highest_sack(sk);
1987	if (!skb)
1988	break;
1989	cache++;
1990	goto walk;
1991	}
1992
1993	skb = tcp_sacktag_skip(skb, sk, skip_to_seq: cache->end_seq);
1994	/* Check overlap against next cached too (past this one already) */
1995	cache++;
1996	continue;
1997	}
1998
1999	if (!before(seq1: start_seq, seq2: tcp_highest_sack_seq(tp))) {
2000	skb = tcp_highest_sack(sk);
2001	if (!skb)
2002	break;
2003	}
2004	skb = tcp_sacktag_skip(skb, sk, skip_to_seq: start_seq);
2005
2006	walk:
2007	skb = tcp_sacktag_walk(skb, sk, next_dup, state,
2008	start_seq, end_seq, dup_sack_in: dup_sack);
2009
2010	advance_sp:
2011	i++;
2012	}
2013
2014	/* Clear the head of the cache sack blocks so we can skip it next time */
2015	for (i = 0; i < ARRAY_SIZE(tp->recv_sack_cache) - used_sacks; i++) {
2016	tp->recv_sack_cache[i].start_seq = 0;
2017	tp->recv_sack_cache[i].end_seq = 0;
2018	}
2019	for (j = 0; j < used_sacks; j++)
2020	tp->recv_sack_cache[i++] = sp[j];
2021
2022	if (inet_csk(sk)->icsk_ca_state != TCP_CA_Loss || tp->undo_marker)
2023	tcp_check_sack_reordering(sk, low_seq: state->reord, ts: 0);
2024
2025	tcp_verify_left_out(tp);
2026	out:
2027
2028	#if FASTRETRANS_DEBUG > 0
2029	WARN_ON((int)tp->sacked_out < 0);
2030	WARN_ON((int)tp->lost_out < 0);
2031	WARN_ON((int)tp->retrans_out < 0);
2032	WARN_ON((int)tcp_packets_in_flight(tp) < 0);
2033	#endif
2034	return state->flag;
2035	}
2036
2037	/* Limits sacked_out so that sum with lost_out isn't ever larger than
2038	* packets_out. Returns false if sacked_out adjustement wasn't necessary.
2039	*/
2040	static bool tcp_limit_reno_sacked(struct tcp_sock *tp)
2041	{
2042	u32 holes;
2043
2044	holes = max(tp->lost_out, 1U);
2045	holes = min(holes, tp->packets_out);
2046
2047	if ((tp->sacked_out + holes) > tp->packets_out) {
2048	tp->sacked_out = tp->packets_out - holes;
2049	return true;
2050	}
2051	return false;
2052	}
2053
2054	/* If we receive more dupacks than we expected counting segments
2055	* in assumption of absent reordering, interpret this as reordering.
2056	* The only another reason could be bug in receiver TCP.
2057	*/
2058	static void tcp_check_reno_reordering(struct sock *sk, const int addend)
2059	{
2060	struct tcp_sock *tp = tcp_sk(sk);
2061
2062	if (!tcp_limit_reno_sacked(tp))
2063	return;
2064
2065	tp->reordering = min_t(u32, tp->packets_out + addend,
2066	READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering));
2067	tp->reord_seen++;
2068	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRENOREORDER);
2069	}
2070
2071	/* Emulate SACKs for SACKless connection: account for a new dupack. */
2072
2073	static void tcp_add_reno_sack(struct sock *sk, int num_dupack, bool ece_ack)
2074	{
2075	if (num_dupack) {
2076	struct tcp_sock *tp = tcp_sk(sk);
2077	u32 prior_sacked = tp->sacked_out;
2078	s32 delivered;
2079
2080	tp->sacked_out += num_dupack;
2081	tcp_check_reno_reordering(sk, addend: 0);
2082	delivered = tp->sacked_out - prior_sacked;
2083	if (delivered > 0)
2084	tcp_count_delivered(tp, delivered, ece_ack);
2085	tcp_verify_left_out(tp);
2086	}
2087	}
2088
2089	/* Account for ACK, ACKing some data in Reno Recovery phase. */
2090
2091	static void tcp_remove_reno_sacks(struct sock *sk, int acked, bool ece_ack)
2092	{
2093	struct tcp_sock *tp = tcp_sk(sk);
2094
2095	if (acked > 0) {
2096	/* One ACK acked hole. The rest eat duplicate ACKs. */
2097	tcp_count_delivered(tp, max_t(int, acked - tp->sacked_out, 1),
2098	ece_ack);
2099	if (acked - 1 >= tp->sacked_out)
2100	tp->sacked_out = 0;
2101	else
2102	tp->sacked_out -= acked - 1;
2103	}
2104	tcp_check_reno_reordering(sk, addend: acked);
2105	tcp_verify_left_out(tp);
2106	}
2107
2108	static inline void tcp_reset_reno_sack(struct tcp_sock *tp)
2109	{
2110	tp->sacked_out = 0;
2111	}
2112
2113	void tcp_clear_retrans(struct tcp_sock *tp)
2114	{
2115	tp->retrans_out = 0;
2116	tp->lost_out = 0;
2117	tp->undo_marker = 0;
2118	tp->undo_retrans = -1;
2119	tp->sacked_out = 0;
2120	tp->rto_stamp = 0;
2121	tp->total_rto = 0;
2122	tp->total_rto_recoveries = 0;
2123	tp->total_rto_time = 0;
2124	}
2125
2126	static inline void tcp_init_undo(struct tcp_sock *tp)
2127	{
2128	tp->undo_marker = tp->snd_una;
2129	/* Retransmission still in flight may cause DSACKs later. */
2130	tp->undo_retrans = tp->retrans_out ? : -1;
2131	}
2132
2133	static bool tcp_is_rack(const struct sock *sk)
2134	{
2135	return READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_recovery) &
2136	TCP_RACK_LOSS_DETECTION;
2137	}
2138
2139	/* If we detect SACK reneging, forget all SACK information
2140	* and reset tags completely, otherwise preserve SACKs. If receiver
2141	* dropped its ofo queue, we will know this due to reneging detection.
2142	*/
2143	static void tcp_timeout_mark_lost(struct sock *sk)
2144	{
2145	struct tcp_sock *tp = tcp_sk(sk);
2146	struct sk_buff *skb, *head;
2147	bool is_reneg; /* is receiver reneging on SACKs? */
2148
2149	head = tcp_rtx_queue_head(sk);
2150	is_reneg = head && (TCP_SKB_CB(head)->sacked & TCPCB_SACKED_ACKED);
2151	if (is_reneg) {
2152	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSACKRENEGING);
2153	tp->sacked_out = 0;
2154	/* Mark SACK reneging until we recover from this loss event. */
2155	tp->is_sack_reneg = 1;
2156	} else if (tcp_is_reno(tp)) {
2157	tcp_reset_reno_sack(tp);
2158	}
2159
2160	skb = head;
2161	skb_rbtree_walk_from(skb) {
2162	if (is_reneg)
2163	TCP_SKB_CB(skb)->sacked &= ~TCPCB_SACKED_ACKED;
2164	else if (tcp_is_rack(sk) && skb != head &&
2165	tcp_rack_skb_timeout(tp, skb, reo_wnd: 0) > 0)
2166	continue; /* Don't mark recently sent ones lost yet */
2167	tcp_mark_skb_lost(sk, skb);
2168	}
2169	tcp_verify_left_out(tp);
2170	tcp_clear_all_retrans_hints(tp);
2171	}
2172
2173	/* Enter Loss state. */
2174	void tcp_enter_loss(struct sock *sk)
2175	{
2176	const struct inet_connection_sock *icsk = inet_csk(sk);
2177	struct tcp_sock *tp = tcp_sk(sk);
2178	struct net *net = sock_net(sk);
2179	bool new_recovery = icsk->icsk_ca_state < TCP_CA_Recovery;
2180	u8 reordering;
2181
2182	tcp_timeout_mark_lost(sk);
2183
2184	/* Reduce ssthresh if it has not yet been made inside this window. */
2185	if (icsk->icsk_ca_state <= TCP_CA_Disorder ||
2186	!after(tp->high_seq, tp->snd_una) ||
2187	(icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) {
2188	tp->prior_ssthresh = tcp_current_ssthresh(sk);
2189	tp->prior_cwnd = tcp_snd_cwnd(tp);
2190	tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk);
2191	tcp_ca_event(sk, event: CA_EVENT_LOSS);
2192	tcp_init_undo(tp);
2193	}
2194	tcp_snd_cwnd_set(tp, val: tcp_packets_in_flight(tp) + 1);
2195	tp->snd_cwnd_cnt = 0;
2196	tp->snd_cwnd_stamp = tcp_jiffies32;
2197
2198	/* Timeout in disordered state after receiving substantial DUPACKs
2199	* suggests that the degree of reordering is over-estimated.
2200	*/
2201	reordering = READ_ONCE(net->ipv4.sysctl_tcp_reordering);
2202	if (icsk->icsk_ca_state <= TCP_CA_Disorder &&
2203	tp->sacked_out >= reordering)
2204	tp->reordering = min_t(unsigned int, tp->reordering,
2205	reordering);
2206
2207	tcp_set_ca_state(sk, ca_state: TCP_CA_Loss);
2208	tp->high_seq = tp->snd_nxt;
2209	tcp_ecn_queue_cwr(tp);
2210
2211	/* F-RTO RFC5682 sec 3.1 step 1: retransmit SND.UNA if no previous
2212	* loss recovery is underway except recurring timeout(s) on
2213	* the same SND.UNA (sec 3.2). Disable F-RTO on path MTU probing
2214	*/
2215	tp->frto = READ_ONCE(net->ipv4.sysctl_tcp_frto) &&
2216	(new_recovery || icsk->icsk_retransmits) &&
2217	!inet_csk(sk)->icsk_mtup.probe_size;
2218	}
2219
2220	/* If ACK arrived pointing to a remembered SACK, it means that our
2221	* remembered SACKs do not reflect real state of receiver i.e.
2222	* receiver _host_ is heavily congested (or buggy).
2223	*
2224	* To avoid big spurious retransmission bursts due to transient SACK
2225	* scoreboard oddities that look like reneging, we give the receiver a
2226	* little time (max(RTT/2, 10ms)) to send us some more ACKs that will
2227	* restore sanity to the SACK scoreboard. If the apparent reneging
2228	* persists until this RTO then we'll clear the SACK scoreboard.
2229	*/
2230	static bool tcp_check_sack_reneging(struct sock *sk, int *ack_flag)
2231	{
2232	if (*ack_flag & FLAG_SACK_RENEGING &&
2233	*ack_flag & FLAG_SND_UNA_ADVANCED) {
2234	struct tcp_sock *tp = tcp_sk(sk);
2235	unsigned long delay = max(usecs_to_jiffies(tp->srtt_us >> 4),
2236	msecs_to_jiffies(10));
2237
2238	inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2239	when: delay, TCP_RTO_MAX);
2240	*ack_flag &= ~FLAG_SET_XMIT_TIMER;
2241	return true;
2242	}
2243	return false;
2244	}
2245
2246	/* Heurestics to calculate number of duplicate ACKs. There's no dupACKs
2247	* counter when SACK is enabled (without SACK, sacked_out is used for
2248	* that purpose).
2249	*
2250	* With reordering, holes may still be in flight, so RFC3517 recovery
2251	* uses pure sacked_out (total number of SACKed segments) even though
2252	* it violates the RFC that uses duplicate ACKs, often these are equal
2253	* but when e.g. out-of-window ACKs or packet duplication occurs,
2254	* they differ. Since neither occurs due to loss, TCP should really
2255	* ignore them.
2256	*/
2257	static inline int tcp_dupack_heuristics(const struct tcp_sock *tp)
2258	{
2259	return tp->sacked_out + 1;
2260	}
2261
2262	/* Linux NewReno/SACK/ECN state machine.
2263	* --------------------------------------
2264	*
2265	* "Open" Normal state, no dubious events, fast path.
2266	* "Disorder" In all the respects it is "Open",
2267	* but requires a bit more attention. It is entered when
2268	* we see some SACKs or dupacks. It is split of "Open"
2269	* mainly to move some processing from fast path to slow one.
2270	* "CWR" CWND was reduced due to some Congestion Notification event.
2271	* It can be ECN, ICMP source quench, local device congestion.
2272	* "Recovery" CWND was reduced, we are fast-retransmitting.
2273	* "Loss" CWND was reduced due to RTO timeout or SACK reneging.
2274	*
2275	* tcp_fastretrans_alert() is entered:
2276	* - each incoming ACK, if state is not "Open"
2277	* - when arrived ACK is unusual, namely:
2278	* * SACK
2279	* * Duplicate ACK.
2280	* * ECN ECE.
2281	*
2282	* Counting packets in flight is pretty simple.
2283	*
2284	* in_flight = packets_out - left_out + retrans_out
2285	*
2286	* packets_out is SND.NXT-SND.UNA counted in packets.
2287	*
2288	* retrans_out is number of retransmitted segments.
2289	*
2290	* left_out is number of segments left network, but not ACKed yet.
2291	*
2292	* left_out = sacked_out + lost_out
2293	*
2294	* sacked_out: Packets, which arrived to receiver out of order
2295	* and hence not ACKed. With SACKs this number is simply
2296	* amount of SACKed data. Even without SACKs
2297	* it is easy to give pretty reliable estimate of this number,
2298	* counting duplicate ACKs.
2299	*
2300	* lost_out: Packets lost by network. TCP has no explicit
2301	* "loss notification" feedback from network (for now).
2302	* It means that this number can be only _guessed_.
2303	* Actually, it is the heuristics to predict lossage that
2304	* distinguishes different algorithms.
2305	*
2306	* F.e. after RTO, when all the queue is considered as lost,
2307	* lost_out = packets_out and in_flight = retrans_out.
2308	*
2309	* Essentially, we have now a few algorithms detecting
2310	* lost packets.
2311	*
2312	* If the receiver supports SACK:
2313	*
2314	* RFC6675/3517: It is the conventional algorithm. A packet is
2315	* considered lost if the number of higher sequence packets
2316	* SACKed is greater than or equal the DUPACK thoreshold
2317	* (reordering). This is implemented in tcp_mark_head_lost and
2318	* tcp_update_scoreboard.
2319	*
2320	* RACK (draft-ietf-tcpm-rack-01): it is a newer algorithm
2321	* (2017-) that checks timing instead of counting DUPACKs.
2322	* Essentially a packet is considered lost if it's not S/ACKed
2323	* after RTT + reordering_window, where both metrics are
2324	* dynamically measured and adjusted. This is implemented in
2325	* tcp_rack_mark_lost.
2326	*
2327	* If the receiver does not support SACK:
2328	*
2329	* NewReno (RFC6582): in Recovery we assume that one segment
2330	* is lost (classic Reno). While we are in Recovery and
2331	* a partial ACK arrives, we assume that one more packet
2332	* is lost (NewReno). This heuristics are the same in NewReno
2333	* and SACK.
2334	*
2335	* Really tricky (and requiring careful tuning) part of algorithm
2336	* is hidden in functions tcp_time_to_recover() and tcp_xmit_retransmit_queue().
2337	* The first determines the moment _when_ we should reduce CWND and,
2338	* hence, slow down forward transmission. In fact, it determines the moment
2339	* when we decide that hole is caused by loss, rather than by a reorder.
2340	*
2341	* tcp_xmit_retransmit_queue() decides, _what_ we should retransmit to fill
2342	* holes, caused by lost packets.
2343	*
2344	* And the most logically complicated part of algorithm is undo
2345	* heuristics. We detect false retransmits due to both too early
2346	* fast retransmit (reordering) and underestimated RTO, analyzing
2347	* timestamps and D-SACKs. When we detect that some segments were
2348	* retransmitted by mistake and CWND reduction was wrong, we undo
2349	* window reduction and abort recovery phase. This logic is hidden
2350	* inside several functions named tcp_try_undo_<something>.
2351	*/
2352
2353	/* This function decides, when we should leave Disordered state
2354	* and enter Recovery phase, reducing congestion window.
2355	*
2356	* Main question: may we further continue forward transmission
2357	* with the same cwnd?
2358	*/
2359	static bool tcp_time_to_recover(struct sock *sk, int flag)
2360	{
2361	struct tcp_sock *tp = tcp_sk(sk);
2362
2363	/* Trick#1: The loss is proven. */
2364	if (tp->lost_out)
2365	return true;
2366
2367	/* Not-A-Trick#2 : Classic rule... */
2368	if (!tcp_is_rack(sk) && tcp_dupack_heuristics(tp) > tp->reordering)
2369	return true;
2370
2371	return false;
2372	}
2373
2374	/* Detect loss in event "A" above by marking head of queue up as lost.
2375	* For RFC3517 SACK, a segment is considered lost if it
2376	* has at least tp->reordering SACKed seqments above it; "packets" refers to
2377	* the maximum SACKed segments to pass before reaching this limit.
2378	*/
2379	static void tcp_mark_head_lost(struct sock *sk, int packets, int mark_head)
2380	{
2381	struct tcp_sock *tp = tcp_sk(sk);
2382	struct sk_buff *skb;
2383	int cnt;
2384	/* Use SACK to deduce losses of new sequences sent during recovery */
2385	const u32 loss_high = tp->snd_nxt;
2386
2387	WARN_ON(packets > tp->packets_out);
2388	skb = tp->lost_skb_hint;
2389	if (skb) {
2390	/* Head already handled? */
2391	if (mark_head && after(TCP_SKB_CB(skb)->seq, tp->snd_una))
2392	return;
2393	cnt = tp->lost_cnt_hint;
2394	} else {
2395	skb = tcp_rtx_queue_head(sk);
2396	cnt = 0;
2397	}
2398
2399	skb_rbtree_walk_from(skb) {
2400	/* TODO: do this better */
2401	/* this is not the most efficient way to do this... */
2402	tp->lost_skb_hint = skb;
2403	tp->lost_cnt_hint = cnt;
2404
2405	if (after(TCP_SKB_CB(skb)->end_seq, loss_high))
2406	break;
2407
2408	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
2409	cnt += tcp_skb_pcount(skb);
2410
2411	if (cnt > packets)
2412	break;
2413
2414	if (!(TCP_SKB_CB(skb)->sacked & TCPCB_LOST))
2415	tcp_mark_skb_lost(sk, skb);
2416
2417	if (mark_head)
2418	break;
2419	}
2420	tcp_verify_left_out(tp);
2421	}
2422
2423	/* Account newly detected lost packet(s) */
2424
2425	static void tcp_update_scoreboard(struct sock *sk, int fast_rexmit)
2426	{
2427	struct tcp_sock *tp = tcp_sk(sk);
2428
2429	if (tcp_is_sack(tp)) {
2430	int sacked_upto = tp->sacked_out - tp->reordering;
2431	if (sacked_upto >= 0)
2432	tcp_mark_head_lost(sk, packets: sacked_upto, mark_head: 0);
2433	else if (fast_rexmit)
2434	tcp_mark_head_lost(sk, packets: 1, mark_head: 1);
2435	}
2436	}
2437
2438	static bool tcp_tsopt_ecr_before(const struct tcp_sock *tp, u32 when)
2439	{
2440	return tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
2441	before(seq1: tp->rx_opt.rcv_tsecr, seq2: when);
2442	}
2443
2444	/* skb is spurious retransmitted if the returned timestamp echo
2445	* reply is prior to the skb transmission time
2446	*/
2447	static bool tcp_skb_spurious_retrans(const struct tcp_sock *tp,
2448	const struct sk_buff *skb)
2449	{
2450	return (TCP_SKB_CB(skb)->sacked & TCPCB_RETRANS) &&
2451	tcp_tsopt_ecr_before(tp, when: tcp_skb_timestamp_ts(usec_ts: tp->tcp_usec_ts, skb));
2452	}
2453
2454	/* Nothing was retransmitted or returned timestamp is less
2455	* than timestamp of the first retransmission.
2456	*/
2457	static inline bool tcp_packet_delayed(const struct tcp_sock *tp)
2458	{
2459	return tp->retrans_stamp &&
2460	tcp_tsopt_ecr_before(tp, when: tp->retrans_stamp);
2461	}
2462
2463	/* Undo procedures. */
2464
2465	/* We can clear retrans_stamp when there are no retransmissions in the
2466	* window. It would seem that it is trivially available for us in
2467	* tp->retrans_out, however, that kind of assumptions doesn't consider
2468	* what will happen if errors occur when sending retransmission for the
2469	* second time. ...It could the that such segment has only
2470	* TCPCB_EVER_RETRANS set at the present time. It seems that checking
2471	* the head skb is enough except for some reneging corner cases that
2472	* are not worth the effort.
2473	*
2474	* Main reason for all this complexity is the fact that connection dying
2475	* time now depends on the validity of the retrans_stamp, in particular,
2476	* that successive retransmissions of a segment must not advance
2477	* retrans_stamp under any conditions.
2478	*/
2479	static bool tcp_any_retrans_done(const struct sock *sk)
2480	{
2481	const struct tcp_sock *tp = tcp_sk(sk);
2482	struct sk_buff *skb;
2483
2484	if (tp->retrans_out)
2485	return true;
2486
2487	skb = tcp_rtx_queue_head(sk);
2488	if (unlikely(skb && TCP_SKB_CB(skb)->sacked & TCPCB_EVER_RETRANS))
2489	return true;
2490
2491	return false;
2492	}
2493
2494	static void DBGUNDO(struct sock *sk, const char *msg)
2495	{
2496	#if FASTRETRANS_DEBUG > 1
2497	struct tcp_sock *tp = tcp_sk(sk);
2498	struct inet_sock *inet = inet_sk(sk);
2499
2500	if (sk->sk_family == AF_INET) {
2501	pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n",
2502	msg,
2503	&inet->inet_daddr, ntohs(inet->inet_dport),
2504	tcp_snd_cwnd(tp), tcp_left_out(tp),
2505	tp->snd_ssthresh, tp->prior_ssthresh,
2506	tp->packets_out);
2507	}
2508	#if IS_ENABLED(CONFIG_IPV6)
2509	else if (sk->sk_family == AF_INET6) {
2510	pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n",
2511	msg,
2512	&sk->sk_v6_daddr, ntohs(inet->inet_dport),
2513	tcp_snd_cwnd(tp), tcp_left_out(tp),
2514	tp->snd_ssthresh, tp->prior_ssthresh,
2515	tp->packets_out);
2516	}
2517	#endif
2518	#endif
2519	}
2520
2521	static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss)
2522	{
2523	struct tcp_sock *tp = tcp_sk(sk);
2524
2525	if (unmark_loss) {
2526	struct sk_buff *skb;
2527
2528	skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
2529	TCP_SKB_CB(skb)->sacked &= ~TCPCB_LOST;
2530	}
2531	tp->lost_out = 0;
2532	tcp_clear_all_retrans_hints(tp);
2533	}
2534
2535	if (tp->prior_ssthresh) {
2536	const struct inet_connection_sock *icsk = inet_csk(sk);
2537
2538	tcp_snd_cwnd_set(tp, val: icsk->icsk_ca_ops->undo_cwnd(sk));
2539
2540	if (tp->prior_ssthresh > tp->snd_ssthresh) {
2541	tp->snd_ssthresh = tp->prior_ssthresh;
2542	tcp_ecn_withdraw_cwr(tp);
2543	}
2544	}
2545	tp->snd_cwnd_stamp = tcp_jiffies32;
2546	tp->undo_marker = 0;
2547	tp->rack.advanced = 1; /* Force RACK to re-exam losses */
2548	}
2549
2550	static inline bool tcp_may_undo(const struct tcp_sock *tp)
2551	{
2552	return tp->undo_marker && (!tp->undo_retrans || tcp_packet_delayed(tp));
2553	}
2554
2555	static bool tcp_is_non_sack_preventing_reopen(struct sock *sk)
2556	{
2557	struct tcp_sock *tp = tcp_sk(sk);
2558
2559	if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) {
2560	/* Hold old state until something *above* high_seq
2561	* is ACKed. For Reno it is MUST to prevent false
2562	* fast retransmits (RFC2582). SACK TCP is safe. */
2563	if (!tcp_any_retrans_done(sk))
2564	tp->retrans_stamp = 0;
2565	return true;
2566	}
2567	return false;
2568	}
2569
2570	/* People celebrate: "We love our President!" */
2571	static bool tcp_try_undo_recovery(struct sock *sk)
2572	{
2573	struct tcp_sock *tp = tcp_sk(sk);
2574
2575	if (tcp_may_undo(tp)) {
2576	int mib_idx;
2577
2578	/* Happy end! We did not retransmit anything
2579	* or our original transmission succeeded.
2580	*/
2581	DBGUNDO(sk, msg: inet_csk(sk)->icsk_ca_state == TCP_CA_Loss ? "loss" : "retrans");
2582	tcp_undo_cwnd_reduction(sk, unmark_loss: false);
2583	if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss)
2584	mib_idx = LINUX_MIB_TCPLOSSUNDO;
2585	else
2586	mib_idx = LINUX_MIB_TCPFULLUNDO;
2587
2588	NET_INC_STATS(sock_net(sk), mib_idx);
2589	} else if (tp->rack.reo_wnd_persist) {
2590	tp->rack.reo_wnd_persist--;
2591	}
2592	if (tcp_is_non_sack_preventing_reopen(sk))
2593	return true;
2594	tcp_set_ca_state(sk, ca_state: TCP_CA_Open);
2595	tp->is_sack_reneg = 0;
2596	return false;
2597	}
2598
2599	/* Try to undo cwnd reduction, because D-SACKs acked all retransmitted data */
2600	static bool tcp_try_undo_dsack(struct sock *sk)
2601	{
2602	struct tcp_sock *tp = tcp_sk(sk);
2603
2604	if (tp->undo_marker && !tp->undo_retrans) {
2605	tp->rack.reo_wnd_persist = min(TCP_RACK_RECOVERY_THRESH,
2606	tp->rack.reo_wnd_persist + 1);
2607	DBGUNDO(sk, msg: "D-SACK");
2608	tcp_undo_cwnd_reduction(sk, unmark_loss: false);
2609	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKUNDO);
2610	return true;
2611	}
2612	return false;
2613	}
2614
2615	/* Undo during loss recovery after partial ACK or using F-RTO. */
2616	static bool tcp_try_undo_loss(struct sock *sk, bool frto_undo)
2617	{
2618	struct tcp_sock *tp = tcp_sk(sk);
2619
2620	if (frto_undo || tcp_may_undo(tp)) {
2621	tcp_undo_cwnd_reduction(sk, unmark_loss: true);
2622
2623	DBGUNDO(sk, msg: "partial loss");
2624	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSUNDO);
2625	if (frto_undo)
2626	NET_INC_STATS(sock_net(sk),
2627	LINUX_MIB_TCPSPURIOUSRTOS);
2628	inet_csk(sk)->icsk_retransmits = 0;
2629	if (tcp_is_non_sack_preventing_reopen(sk))
2630	return true;
2631	if (frto_undo || tcp_is_sack(tp)) {
2632	tcp_set_ca_state(sk, ca_state: TCP_CA_Open);
2633	tp->is_sack_reneg = 0;
2634	}
2635	return true;
2636	}
2637	return false;
2638	}
2639
2640	/* The cwnd reduction in CWR and Recovery uses the PRR algorithm in RFC 6937.
2641	* It computes the number of packets to send (sndcnt) based on packets newly
2642	* delivered:
2643	* 1) If the packets in flight is larger than ssthresh, PRR spreads the
2644	* cwnd reductions across a full RTT.
2645	* 2) Otherwise PRR uses packet conservation to send as much as delivered.
2646	* But when SND_UNA is acked without further losses,
2647	* slow starts cwnd up to ssthresh to speed up the recovery.
2648	*/
2649	static void tcp_init_cwnd_reduction(struct sock *sk)
2650	{
2651	struct tcp_sock *tp = tcp_sk(sk);
2652
2653	tp->high_seq = tp->snd_nxt;
2654	tp->tlp_high_seq = 0;
2655	tp->snd_cwnd_cnt = 0;
2656	tp->prior_cwnd = tcp_snd_cwnd(tp);
2657	tp->prr_delivered = 0;
2658	tp->prr_out = 0;
2659	tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk);
2660	tcp_ecn_queue_cwr(tp);
2661	}
2662
2663	void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag)
2664	{
2665	struct tcp_sock *tp = tcp_sk(sk);
2666	int sndcnt = 0;
2667	int delta = tp->snd_ssthresh - tcp_packets_in_flight(tp);
2668
2669	if (newly_acked_sacked <= 0 || WARN_ON_ONCE(!tp->prior_cwnd))
2670	return;
2671
2672	tp->prr_delivered += newly_acked_sacked;
2673	if (delta < 0) {
2674	u64 dividend = (u64)tp->snd_ssthresh * tp->prr_delivered +
2675	tp->prior_cwnd - 1;
2676	sndcnt = div_u64(dividend, divisor: tp->prior_cwnd) - tp->prr_out;
2677	} else {
2678	sndcnt = max_t(int, tp->prr_delivered - tp->prr_out,
2679	newly_acked_sacked);
2680	if (flag & FLAG_SND_UNA_ADVANCED && !newly_lost)
2681	sndcnt++;
2682	sndcnt = min(delta, sndcnt);
2683	}
2684	/* Force a fast retransmit upon entering fast recovery */
2685	sndcnt = max(sndcnt, (tp->prr_out ? 0 : 1));
2686	tcp_snd_cwnd_set(tp, val: tcp_packets_in_flight(tp) + sndcnt);
2687	}
2688
2689	static inline void tcp_end_cwnd_reduction(struct sock *sk)
2690	{
2691	struct tcp_sock *tp = tcp_sk(sk);
2692
2693	if (inet_csk(sk)->icsk_ca_ops->cong_control)
2694	return;
2695
2696	/* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */
2697	if (tp->snd_ssthresh < TCP_INFINITE_SSTHRESH &&
2698	(inet_csk(sk)->icsk_ca_state == TCP_CA_CWR || tp->undo_marker)) {
2699	tcp_snd_cwnd_set(tp, val: tp->snd_ssthresh);
2700	tp->snd_cwnd_stamp = tcp_jiffies32;
2701	}
2702	tcp_ca_event(sk, event: CA_EVENT_COMPLETE_CWR);
2703	}
2704
2705	/* Enter CWR state. Disable cwnd undo since congestion is proven with ECN */
2706	void tcp_enter_cwr(struct sock *sk)
2707	{
2708	struct tcp_sock *tp = tcp_sk(sk);
2709
2710	tp->prior_ssthresh = 0;
2711	if (inet_csk(sk)->icsk_ca_state < TCP_CA_CWR) {
2712	tp->undo_marker = 0;
2713	tcp_init_cwnd_reduction(sk);
2714	tcp_set_ca_state(sk, ca_state: TCP_CA_CWR);
2715	}
2716	}
2717	EXPORT_SYMBOL(tcp_enter_cwr);
2718
2719	static void tcp_try_keep_open(struct sock *sk)
2720	{
2721	struct tcp_sock *tp = tcp_sk(sk);
2722	int state = TCP_CA_Open;
2723
2724	if (tcp_left_out(tp) || tcp_any_retrans_done(sk))
2725	state = TCP_CA_Disorder;
2726
2727	if (inet_csk(sk)->icsk_ca_state != state) {
2728	tcp_set_ca_state(sk, ca_state: state);
2729	tp->high_seq = tp->snd_nxt;
2730	}
2731	}
2732
2733	static void tcp_try_to_open(struct sock *sk, int flag)
2734	{
2735	struct tcp_sock *tp = tcp_sk(sk);
2736
2737	tcp_verify_left_out(tp);
2738
2739	if (!tcp_any_retrans_done(sk))
2740	tp->retrans_stamp = 0;
2741
2742	if (flag & FLAG_ECE)
2743	tcp_enter_cwr(sk);
2744
2745	if (inet_csk(sk)->icsk_ca_state != TCP_CA_CWR) {
2746	tcp_try_keep_open(sk);
2747	}
2748	}
2749
2750	static void tcp_mtup_probe_failed(struct sock *sk)
2751	{
2752	struct inet_connection_sock *icsk = inet_csk(sk);
2753
2754	icsk->icsk_mtup.search_high = icsk->icsk_mtup.probe_size - 1;
2755	icsk->icsk_mtup.probe_size = 0;
2756	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPFAIL);
2757	}
2758
2759	static void tcp_mtup_probe_success(struct sock *sk)
2760	{
2761	struct tcp_sock *tp = tcp_sk(sk);
2762	struct inet_connection_sock *icsk = inet_csk(sk);
2763	u64 val;
2764
2765	tp->prior_ssthresh = tcp_current_ssthresh(sk);
2766
2767	val = (u64)tcp_snd_cwnd(tp) * tcp_mss_to_mtu(sk, mss: tp->mss_cache);
2768	do_div(val, icsk->icsk_mtup.probe_size);
2769	DEBUG_NET_WARN_ON_ONCE((u32)val != val);
2770	tcp_snd_cwnd_set(tp, max_t(u32, 1U, val));
2771
2772	tp->snd_cwnd_cnt = 0;
2773	tp->snd_cwnd_stamp = tcp_jiffies32;
2774	tp->snd_ssthresh = tcp_current_ssthresh(sk);
2775
2776	icsk->icsk_mtup.search_low = icsk->icsk_mtup.probe_size;
2777	icsk->icsk_mtup.probe_size = 0;
2778	tcp_sync_mss(sk, pmtu: icsk->icsk_pmtu_cookie);
2779	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMTUPSUCCESS);
2780	}
2781
2782	/* Do a simple retransmit without using the backoff mechanisms in
2783	* tcp_timer. This is used for path mtu discovery.
2784	* The socket is already locked here.
2785	*/
2786	void tcp_simple_retransmit(struct sock *sk)
2787	{
2788	const struct inet_connection_sock *icsk = inet_csk(sk);
2789	struct tcp_sock *tp = tcp_sk(sk);
2790	struct sk_buff *skb;
2791	int mss;
2792
2793	/* A fastopen SYN request is stored as two separate packets within
2794	* the retransmit queue, this is done by tcp_send_syn_data().
2795	* As a result simply checking the MSS of the frames in the queue
2796	* will not work for the SYN packet.
2797	*
2798	* Us being here is an indication of a path MTU issue so we can
2799	* assume that the fastopen SYN was lost and just mark all the
2800	* frames in the retransmit queue as lost. We will use an MSS of
2801	* -1 to mark all frames as lost, otherwise compute the current MSS.
2802	*/
2803	if (tp->syn_data && sk->sk_state == TCP_SYN_SENT)
2804	mss = -1;
2805	else
2806	mss = tcp_current_mss(sk);
2807
2808	skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
2809	if (tcp_skb_seglen(skb) > mss)
2810	tcp_mark_skb_lost(sk, skb);
2811	}
2812
2813	tcp_clear_retrans_hints_partial(tp);
2814
2815	if (!tp->lost_out)
2816	return;
2817
2818	if (tcp_is_reno(tp))
2819	tcp_limit_reno_sacked(tp);
2820
2821	tcp_verify_left_out(tp);
2822
2823	/* Don't muck with the congestion window here.
2824	* Reason is that we do not increase amount of _data_
2825	* in network, but units changed and effective
2826	* cwnd/ssthresh really reduced now.
2827	*/
2828	if (icsk->icsk_ca_state != TCP_CA_Loss) {
2829	tp->high_seq = tp->snd_nxt;
2830	tp->snd_ssthresh = tcp_current_ssthresh(sk);
2831	tp->prior_ssthresh = 0;
2832	tp->undo_marker = 0;
2833	tcp_set_ca_state(sk, ca_state: TCP_CA_Loss);
2834	}
2835	tcp_xmit_retransmit_queue(sk);
2836	}
2837	EXPORT_SYMBOL(tcp_simple_retransmit);
2838
2839	void tcp_enter_recovery(struct sock *sk, bool ece_ack)
2840	{
2841	struct tcp_sock *tp = tcp_sk(sk);
2842	int mib_idx;
2843
2844	if (tcp_is_reno(tp))
2845	mib_idx = LINUX_MIB_TCPRENORECOVERY;
2846	else
2847	mib_idx = LINUX_MIB_TCPSACKRECOVERY;
2848
2849	NET_INC_STATS(sock_net(sk), mib_idx);
2850
2851	tp->prior_ssthresh = 0;
2852	tcp_init_undo(tp);
2853
2854	if (!tcp_in_cwnd_reduction(sk)) {
2855	if (!ece_ack)
2856	tp->prior_ssthresh = tcp_current_ssthresh(sk);
2857	tcp_init_cwnd_reduction(sk);
2858	}
2859	tcp_set_ca_state(sk, ca_state: TCP_CA_Recovery);
2860	}
2861
2862	static void tcp_update_rto_time(struct tcp_sock *tp)
2863	{
2864	if (tp->rto_stamp) {
2865	tp->total_rto_time += tcp_time_stamp_ms(tp) - tp->rto_stamp;
2866	tp->rto_stamp = 0;
2867	}
2868	}
2869
2870	/* Process an ACK in CA_Loss state. Move to CA_Open if lost data are
2871	* recovered or spurious. Otherwise retransmits more on partial ACKs.
2872	*/
2873	static void tcp_process_loss(struct sock *sk, int flag, int num_dupack,
2874	int *rexmit)
2875	{
2876	struct tcp_sock *tp = tcp_sk(sk);
2877	bool recovered = !before(seq1: tp->snd_una, seq2: tp->high_seq);
2878
2879	if ((flag & FLAG_SND_UNA_ADVANCED || rcu_access_pointer(tp->fastopen_rsk)) &&
2880	tcp_try_undo_loss(sk, frto_undo: false))
2881	return;
2882
2883	if (tp->frto) { /* F-RTO RFC5682 sec 3.1 (sack enhanced version). */
2884	/* Step 3.b. A timeout is spurious if not all data are
2885	* lost, i.e., never-retransmitted data are (s)acked.
2886	*/
2887	if ((flag & FLAG_ORIG_SACK_ACKED) &&
2888	tcp_try_undo_loss(sk, frto_undo: true))
2889	return;
2890
2891	if (after(tp->snd_nxt, tp->high_seq)) {
2892	if (flag & FLAG_DATA_SACKED || num_dupack)
2893	tp->frto = 0; /* Step 3.a. loss was real */
2894	} else if (flag & FLAG_SND_UNA_ADVANCED && !recovered) {
2895	tp->high_seq = tp->snd_nxt;
2896	/* Step 2.b. Try send new data (but deferred until cwnd
2897	* is updated in tcp_ack()). Otherwise fall back to
2898	* the conventional recovery.
2899	*/
2900	if (!tcp_write_queue_empty(sk) &&
2901	after(tcp_wnd_end(tp), tp->snd_nxt)) {
2902	*rexmit = REXMIT_NEW;
2903	return;
2904	}
2905	tp->frto = 0;
2906	}
2907	}
2908
2909	if (recovered) {
2910	/* F-RTO RFC5682 sec 3.1 step 2.a and 1st part of step 3.a */
2911	tcp_try_undo_recovery(sk);
2912	return;
2913	}
2914	if (tcp_is_reno(tp)) {
2915	/* A Reno DUPACK means new data in F-RTO step 2.b above are
2916	* delivered. Lower inflight to clock out (re)transmissions.
2917	*/
2918	if (after(tp->snd_nxt, tp->high_seq) && num_dupack)
2919	tcp_add_reno_sack(sk, num_dupack, ece_ack: flag & FLAG_ECE);
2920	else if (flag & FLAG_SND_UNA_ADVANCED)
2921	tcp_reset_reno_sack(tp);
2922	}
2923	*rexmit = REXMIT_LOST;
2924	}
2925
2926	static bool tcp_force_fast_retransmit(struct sock *sk)
2927	{
2928	struct tcp_sock *tp = tcp_sk(sk);
2929
2930	return after(tcp_highest_sack_seq(tp),
2931	tp->snd_una + tp->reordering * tp->mss_cache);
2932	}
2933
2934	/* Undo during fast recovery after partial ACK. */
2935	static bool tcp_try_undo_partial(struct sock *sk, u32 prior_snd_una,
2936	bool *do_lost)
2937	{
2938	struct tcp_sock *tp = tcp_sk(sk);
2939
2940	if (tp->undo_marker && tcp_packet_delayed(tp)) {
2941	/* Plain luck! Hole if filled with delayed
2942	* packet, rather than with a retransmit. Check reordering.
2943	*/
2944	tcp_check_sack_reordering(sk, low_seq: prior_snd_una, ts: 1);
2945
2946	/* We are getting evidence that the reordering degree is higher
2947	* than we realized. If there are no retransmits out then we
2948	* can undo. Otherwise we clock out new packets but do not
2949	* mark more packets lost or retransmit more.
2950	*/
2951	if (tp->retrans_out)
2952	return true;
2953
2954	if (!tcp_any_retrans_done(sk))
2955	tp->retrans_stamp = 0;
2956
2957	DBGUNDO(sk, msg: "partial recovery");
2958	tcp_undo_cwnd_reduction(sk, unmark_loss: true);
2959	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPARTIALUNDO);
2960	tcp_try_keep_open(sk);
2961	} else {
2962	/* Partial ACK arrived. Force fast retransmit. */
2963	*do_lost = tcp_force_fast_retransmit(sk);
2964	}
2965	return false;
2966	}
2967
2968	static void tcp_identify_packet_loss(struct sock *sk, int *ack_flag)
2969	{
2970	struct tcp_sock *tp = tcp_sk(sk);
2971
2972	if (tcp_rtx_queue_empty(sk))
2973	return;
2974
2975	if (unlikely(tcp_is_reno(tp))) {
2976	tcp_newreno_mark_lost(sk, snd_una_advanced: *ack_flag & FLAG_SND_UNA_ADVANCED);
2977	} else if (tcp_is_rack(sk)) {
2978	u32 prior_retrans = tp->retrans_out;
2979
2980	if (tcp_rack_mark_lost(sk))
2981	*ack_flag &= ~FLAG_SET_XMIT_TIMER;
2982	if (prior_retrans > tp->retrans_out)
2983	*ack_flag |= FLAG_LOST_RETRANS;
2984	}
2985	}
2986
2987	/* Process an event, which can update packets-in-flight not trivially.
2988	* Main goal of this function is to calculate new estimate for left_out,
2989	* taking into account both packets sitting in receiver's buffer and
2990	* packets lost by network.
2991	*
2992	* Besides that it updates the congestion state when packet loss or ECN
2993	* is detected. But it does not reduce the cwnd, it is done by the
2994	* congestion control later.
2995	*
2996	* It does _not_ decide what to send, it is made in function
2997	* tcp_xmit_retransmit_queue().
2998	*/
2999	static void tcp_fastretrans_alert(struct sock *sk, const u32 prior_snd_una,
3000	int num_dupack, int *ack_flag, int *rexmit)
3001	{
3002	struct inet_connection_sock *icsk = inet_csk(sk);
3003	struct tcp_sock *tp = tcp_sk(sk);
3004	int fast_rexmit = 0, flag = *ack_flag;
3005	bool ece_ack = flag & FLAG_ECE;
3006	bool do_lost = num_dupack || ((flag & FLAG_DATA_SACKED) &&
3007	tcp_force_fast_retransmit(sk));
3008
3009	if (!tp->packets_out && tp->sacked_out)
3010	tp->sacked_out = 0;
3011
3012	/* Now state machine starts.
3013	* A. ECE, hence prohibit cwnd undoing, the reduction is required. */
3014	if (ece_ack)
3015	tp->prior_ssthresh = 0;
3016
3017	/* B. In all the states check for reneging SACKs. */
3018	if (tcp_check_sack_reneging(sk, ack_flag))
3019	return;
3020
3021	/* C. Check consistency of the current state. */
3022	tcp_verify_left_out(tp);
3023
3024	/* D. Check state exit conditions. State can be terminated
3025	* when high_seq is ACKed. */
3026	if (icsk->icsk_ca_state == TCP_CA_Open) {
3027	WARN_ON(tp->retrans_out != 0 && !tp->syn_data);
3028	tp->retrans_stamp = 0;
3029	} else if (!before(seq1: tp->snd_una, seq2: tp->high_seq)) {
3030	switch (icsk->icsk_ca_state) {
3031	case TCP_CA_CWR:
3032	/* CWR is to be held something *above* high_seq
3033	* is ACKed for CWR bit to reach receiver. */
3034	if (tp->snd_una != tp->high_seq) {
3035	tcp_end_cwnd_reduction(sk);
3036	tcp_set_ca_state(sk, ca_state: TCP_CA_Open);
3037	}
3038	break;
3039
3040	case TCP_CA_Recovery:
3041	if (tcp_is_reno(tp))
3042	tcp_reset_reno_sack(tp);
3043	if (tcp_try_undo_recovery(sk))
3044	return;
3045	tcp_end_cwnd_reduction(sk);
3046	break;
3047	}
3048	}
3049
3050	/* E. Process state. */
3051	switch (icsk->icsk_ca_state) {
3052	case TCP_CA_Recovery:
3053	if (!(flag & FLAG_SND_UNA_ADVANCED)) {
3054	if (tcp_is_reno(tp))
3055	tcp_add_reno_sack(sk, num_dupack, ece_ack);
3056	} else if (tcp_try_undo_partial(sk, prior_snd_una, do_lost: &do_lost))
3057	return;
3058
3059	if (tcp_try_undo_dsack(sk))
3060	tcp_try_keep_open(sk);
3061
3062	tcp_identify_packet_loss(sk, ack_flag);
3063	if (icsk->icsk_ca_state != TCP_CA_Recovery) {
3064	if (!tcp_time_to_recover(sk, flag))
3065	return;
3066	/* Undo reverts the recovery state. If loss is evident,
3067	* starts a new recovery (e.g. reordering then loss);
3068	*/
3069	tcp_enter_recovery(sk, ece_ack);
3070	}
3071	break;
3072	case TCP_CA_Loss:
3073	tcp_process_loss(sk, flag, num_dupack, rexmit);
3074	if (icsk->icsk_ca_state != TCP_CA_Loss)
3075	tcp_update_rto_time(tp);
3076	tcp_identify_packet_loss(sk, ack_flag);
3077	if (!(icsk->icsk_ca_state == TCP_CA_Open ||
3078	(*ack_flag & FLAG_LOST_RETRANS)))
3079	return;
3080	/* Change state if cwnd is undone or retransmits are lost */
3081	fallthrough;
3082	default:
3083	if (tcp_is_reno(tp)) {
3084	if (flag & FLAG_SND_UNA_ADVANCED)
3085	tcp_reset_reno_sack(tp);
3086	tcp_add_reno_sack(sk, num_dupack, ece_ack);
3087	}
3088
3089	if (icsk->icsk_ca_state <= TCP_CA_Disorder)
3090	tcp_try_undo_dsack(sk);
3091
3092	tcp_identify_packet_loss(sk, ack_flag);
3093	if (!tcp_time_to_recover(sk, flag)) {
3094	tcp_try_to_open(sk, flag);
3095	return;
3096	}
3097
3098	/* MTU probe failure: don't reduce cwnd */
3099	if (icsk->icsk_ca_state < TCP_CA_CWR &&
3100	icsk->icsk_mtup.probe_size &&
3101	tp->snd_una == tp->mtu_probe.probe_seq_start) {
3102	tcp_mtup_probe_failed(sk);
3103	/* Restores the reduction we did in tcp_mtup_probe() */
3104	tcp_snd_cwnd_set(tp, val: tcp_snd_cwnd(tp) + 1);
3105	tcp_simple_retransmit(sk);
3106	return;
3107	}
3108
3109	/* Otherwise enter Recovery state */
3110	tcp_enter_recovery(sk, ece_ack);
3111	fast_rexmit = 1;
3112	}
3113
3114	if (!tcp_is_rack(sk) && do_lost)
3115	tcp_update_scoreboard(sk, fast_rexmit);
3116	*rexmit = REXMIT_LOST;
3117	}
3118
3119	static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us, const int flag)
3120	{
3121	u32 wlen = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_rtt_wlen) * HZ;
3122	struct tcp_sock *tp = tcp_sk(sk);
3123
3124	if ((flag & FLAG_ACK_MAYBE_DELAYED) && rtt_us > tcp_min_rtt(tp)) {
3125	/* If the remote keeps returning delayed ACKs, eventually
3126	* the min filter would pick it up and overestimate the
3127	* prop. delay when it expires. Skip suspected delayed ACKs.
3128	*/
3129	return;
3130	}
3131	minmax_running_min(m: &tp->rtt_min, win: wlen, tcp_jiffies32,
3132	meas: rtt_us ? : jiffies_to_usecs(j: 1));
3133	}
3134
3135	static bool tcp_ack_update_rtt(struct sock *sk, const int flag,
3136	long seq_rtt_us, long sack_rtt_us,
3137	long ca_rtt_us, struct rate_sample *rs)
3138	{
3139	const struct tcp_sock *tp = tcp_sk(sk);
3140
3141	/* Prefer RTT measured from ACK's timing to TS-ECR. This is because
3142	* broken middle-boxes or peers may corrupt TS-ECR fields. But
3143	* Karn's algorithm forbids taking RTT if some retransmitted data
3144	* is acked (RFC6298).
3145	*/
3146	if (seq_rtt_us < 0)
3147	seq_rtt_us = sack_rtt_us;
3148
3149	/* RTTM Rule: A TSecr value received in a segment is used to
3150	* update the averaged RTT measurement only if the segment
3151	* acknowledges some new data, i.e., only if it advances the
3152	* left edge of the send window.
3153	* See draft-ietf-tcplw-high-performance-00, section 3.3.
3154	*/
3155	if (seq_rtt_us < 0 && tp->rx_opt.saw_tstamp &&
3156	tp->rx_opt.rcv_tsecr && flag & FLAG_ACKED)
3157	seq_rtt_us = ca_rtt_us = tcp_rtt_tsopt_us(tp);
3158
3159	rs->rtt_us = ca_rtt_us; /* RTT of last (S)ACKed packet (or -1) */
3160	if (seq_rtt_us < 0)
3161	return false;
3162
3163	/* ca_rtt_us >= 0 is counting on the invariant that ca_rtt_us is
3164	* always taken together with ACK, SACK, or TS-opts. Any negative
3165	* values will be skipped with the seq_rtt_us < 0 check above.
3166	*/
3167	tcp_update_rtt_min(sk, rtt_us: ca_rtt_us, flag);
3168	tcp_rtt_estimator(sk, mrtt_us: seq_rtt_us);
3169	tcp_set_rto(sk);
3170
3171	/* RFC6298: only reset backoff on valid RTT measurement. */
3172	inet_csk(sk)->icsk_backoff = 0;
3173	return true;
3174	}
3175
3176	/* Compute time elapsed between (last) SYNACK and the ACK completing 3WHS. */
3177	void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req)
3178	{
3179	struct rate_sample rs;
3180	long rtt_us = -1L;
3181
3182	if (req && !req->num_retrans && tcp_rsk(req)->snt_synack)
3183	rtt_us = tcp_stamp_us_delta(t1: tcp_clock_us(), t0: tcp_rsk(req)->snt_synack);
3184
3185	tcp_ack_update_rtt(sk, FLAG_SYN_ACKED, seq_rtt_us: rtt_us, sack_rtt_us: -1L, ca_rtt_us: rtt_us, rs: &rs);
3186	}
3187
3188
3189	static void tcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
3190	{
3191	const struct inet_connection_sock *icsk = inet_csk(sk);
3192
3193	icsk->icsk_ca_ops->cong_avoid(sk, ack, acked);
3194	tcp_sk(sk)->snd_cwnd_stamp = tcp_jiffies32;
3195	}
3196
3197	/* Restart timer after forward progress on connection.
3198	* RFC2988 recommends to restart timer to now+rto.
3199	*/
3200	void tcp_rearm_rto(struct sock *sk)
3201	{
3202	const struct inet_connection_sock *icsk = inet_csk(sk);
3203	struct tcp_sock *tp = tcp_sk(sk);
3204
3205	/* If the retrans timer is currently being used by Fast Open
3206	* for SYN-ACK retrans purpose, stay put.
3207	*/
3208	if (rcu_access_pointer(tp->fastopen_rsk))
3209	return;
3210
3211	if (!tp->packets_out) {
3212	inet_csk_clear_xmit_timer(sk, ICSK_TIME_RETRANS);
3213	} else {
3214	u32 rto = inet_csk(sk)->icsk_rto;
3215	/* Offset the time elapsed after installing regular RTO */
3216	if (icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
3217	icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
3218	s64 delta_us = tcp_rto_delta_us(sk);
3219	/* delta_us may not be positive if the socket is locked
3220	* when the retrans timer fires and is rescheduled.
3221	*/
3222	rto = usecs_to_jiffies(max_t(int, delta_us, 1));
3223	}
3224	tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, when: rto,
3225	TCP_RTO_MAX);
3226	}
3227	}
3228
3229	/* Try to schedule a loss probe; if that doesn't work, then schedule an RTO. */
3230	static void tcp_set_xmit_timer(struct sock *sk)
3231	{
3232	if (!tcp_schedule_loss_probe(sk, advancing_rto: true))
3233	tcp_rearm_rto(sk);
3234	}
3235
3236	/* If we get here, the whole TSO packet has not been acked. */
3237	static u32 tcp_tso_acked(struct sock *sk, struct sk_buff *skb)
3238	{
3239	struct tcp_sock *tp = tcp_sk(sk);
3240	u32 packets_acked;
3241
3242	BUG_ON(!after(TCP_SKB_CB(skb)->end_seq, tp->snd_una));
3243
3244	packets_acked = tcp_skb_pcount(skb);
3245	if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
3246	return 0;
3247	packets_acked -= tcp_skb_pcount(skb);
3248
3249	if (packets_acked) {
3250	BUG_ON(tcp_skb_pcount(skb) == 0);
3251	BUG_ON(!before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq));
3252	}
3253
3254	return packets_acked;
3255	}
3256
3257	static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb,
3258	const struct sk_buff *ack_skb, u32 prior_snd_una)
3259	{
3260	const struct skb_shared_info *shinfo;
3261
3262	/* Avoid cache line misses to get skb_shinfo() and shinfo->tx_flags */
3263	if (likely(!TCP_SKB_CB(skb)->txstamp_ack))
3264	return;
3265
3266	shinfo = skb_shinfo(skb);
3267	if (!before(seq1: shinfo->tskey, seq2: prior_snd_una) &&
3268	before(seq1: shinfo->tskey, tcp_sk(sk)->snd_una)) {
3269	tcp_skb_tsorted_save(skb) {
3270	__skb_tstamp_tx(orig_skb: skb, ack_skb, NULL, sk, tstype: SCM_TSTAMP_ACK);
3271	} tcp_skb_tsorted_restore(skb);
3272	}
3273	}
3274
3275	/* Remove acknowledged frames from the retransmission queue. If our packet
3276	* is before the ack sequence we can discard it as it's confirmed to have
3277	* arrived at the other end.
3278	*/
3279	static int tcp_clean_rtx_queue(struct sock *sk, const struct sk_buff *ack_skb,
3280	u32 prior_fack, u32 prior_snd_una,
3281	struct tcp_sacktag_state *sack, bool ece_ack)
3282	{
3283	const struct inet_connection_sock *icsk = inet_csk(sk);
3284	u64 first_ackt, last_ackt;
3285	struct tcp_sock *tp = tcp_sk(sk);
3286	u32 prior_sacked = tp->sacked_out;
3287	u32 reord = tp->snd_nxt; /* lowest acked un-retx un-sacked seq */
3288	struct sk_buff *skb, *next;
3289	bool fully_acked = true;
3290	long sack_rtt_us = -1L;
3291	long seq_rtt_us = -1L;
3292	long ca_rtt_us = -1L;
3293	u32 pkts_acked = 0;
3294	bool rtt_update;
3295	int flag = 0;
3296
3297	first_ackt = 0;
3298
3299	for (skb = skb_rb_first(&sk->tcp_rtx_queue); skb; skb = next) {
3300	struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
3301	const u32 start_seq = scb->seq;
3302	u8 sacked = scb->sacked;
3303	u32 acked_pcount;
3304
3305	/* Determine how many packets and what bytes were acked, tso and else */
3306	if (after(scb->end_seq, tp->snd_una)) {
3307	if (tcp_skb_pcount(skb) == 1 ||
3308	!after(tp->snd_una, scb->seq))
3309	break;
3310
3311	acked_pcount = tcp_tso_acked(sk, skb);
3312	if (!acked_pcount)
3313	break;
3314	fully_acked = false;
3315	} else {
3316	acked_pcount = tcp_skb_pcount(skb);
3317	}
3318
3319	if (unlikely(sacked & TCPCB_RETRANS)) {
3320	if (sacked & TCPCB_SACKED_RETRANS)
3321	tp->retrans_out -= acked_pcount;
3322	flag |= FLAG_RETRANS_DATA_ACKED;
3323	} else if (!(sacked & TCPCB_SACKED_ACKED)) {
3324	last_ackt = tcp_skb_timestamp_us(skb);
3325	WARN_ON_ONCE(last_ackt == 0);
3326	if (!first_ackt)
3327	first_ackt = last_ackt;
3328
3329	if (before(seq1: start_seq, seq2: reord))
3330	reord = start_seq;
3331	if (!after(scb->end_seq, tp->high_seq))
3332	flag |= FLAG_ORIG_SACK_ACKED;
3333	}
3334
3335	if (sacked & TCPCB_SACKED_ACKED) {
3336	tp->sacked_out -= acked_pcount;
3337	} else if (tcp_is_sack(tp)) {
3338	tcp_count_delivered(tp, delivered: acked_pcount, ece_ack);
3339	if (!tcp_skb_spurious_retrans(tp, skb))
3340	tcp_rack_advance(tp, sacked, end_seq: scb->end_seq,
3341	xmit_time: tcp_skb_timestamp_us(skb));
3342	}
3343	if (sacked & TCPCB_LOST)
3344	tp->lost_out -= acked_pcount;
3345
3346	tp->packets_out -= acked_pcount;
3347	pkts_acked += acked_pcount;
3348	tcp_rate_skb_delivered(sk, skb, rs: sack->rate);
3349
3350	/* Initial outgoing SYN's get put onto the write_queue
3351	* just like anything else we transmit. It is not
3352	* true data, and if we misinform our callers that
3353	* this ACK acks real data, we will erroneously exit
3354	* connection startup slow start one packet too
3355	* quickly. This is severely frowned upon behavior.
3356	*/
3357	if (likely(!(scb->tcp_flags & TCPHDR_SYN))) {
3358	flag |= FLAG_DATA_ACKED;
3359	} else {
3360	flag |= FLAG_SYN_ACKED;
3361	tp->retrans_stamp = 0;
3362	}
3363
3364	if (!fully_acked)
3365	break;
3366
3367	tcp_ack_tstamp(sk, skb, ack_skb, prior_snd_una);
3368
3369	next = skb_rb_next(skb);
3370	if (unlikely(skb == tp->retransmit_skb_hint))
3371	tp->retransmit_skb_hint = NULL;
3372	if (unlikely(skb == tp->lost_skb_hint))
3373	tp->lost_skb_hint = NULL;
3374	tcp_highest_sack_replace(sk, old: skb, new: next);
3375	tcp_rtx_queue_unlink_and_free(skb, sk);
3376	}
3377
3378	if (!skb)
3379	tcp_chrono_stop(sk, type: TCP_CHRONO_BUSY);
3380
3381	if (likely(between(tp->snd_up, prior_snd_una, tp->snd_una)))
3382	tp->snd_up = tp->snd_una;
3383
3384	if (skb) {
3385	tcp_ack_tstamp(sk, skb, ack_skb, prior_snd_una);
3386	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
3387	flag |= FLAG_SACK_RENEGING;
3388	}
3389
3390	if (likely(first_ackt) && !(flag & FLAG_RETRANS_DATA_ACKED)) {
3391	seq_rtt_us = tcp_stamp_us_delta(t1: tp->tcp_mstamp, t0: first_ackt);
3392	ca_rtt_us = tcp_stamp_us_delta(t1: tp->tcp_mstamp, t0: last_ackt);
3393
3394	if (pkts_acked == 1 && fully_acked && !prior_sacked &&
3395	(tp->snd_una - prior_snd_una) < tp->mss_cache &&
3396	sack->rate->prior_delivered + 1 == tp->delivered &&
3397	!(flag & (FLAG_CA_ALERT | FLAG_SYN_ACKED))) {
3398	/* Conservatively mark a delayed ACK. It's typically
3399	* from a lone runt packet over the round trip to
3400	* a receiver w/o out-of-order or CE events.
3401	*/
3402	flag |= FLAG_ACK_MAYBE_DELAYED;
3403	}
3404	}
3405	if (sack->first_sackt) {
3406	sack_rtt_us = tcp_stamp_us_delta(t1: tp->tcp_mstamp, t0: sack->first_sackt);
3407	ca_rtt_us = tcp_stamp_us_delta(t1: tp->tcp_mstamp, t0: sack->last_sackt);
3408	}
3409	rtt_update = tcp_ack_update_rtt(sk, flag, seq_rtt_us, sack_rtt_us,
3410	ca_rtt_us, rs: sack->rate);
3411
3412	if (flag & FLAG_ACKED) {
3413	flag |= FLAG_SET_XMIT_TIMER; /* set TLP or RTO timer */
3414	if (unlikely(icsk->icsk_mtup.probe_size &&
3415	!after(tp->mtu_probe.probe_seq_end, tp->snd_una))) {
3416	tcp_mtup_probe_success(sk);
3417	}
3418
3419	if (tcp_is_reno(tp)) {
3420	tcp_remove_reno_sacks(sk, acked: pkts_acked, ece_ack);
3421
3422	/* If any of the cumulatively ACKed segments was
3423	* retransmitted, non-SACK case cannot confirm that
3424	* progress was due to original transmission due to
3425	* lack of TCPCB_SACKED_ACKED bits even if some of
3426	* the packets may have been never retransmitted.
3427	*/
3428	if (flag & FLAG_RETRANS_DATA_ACKED)
3429	flag &= ~FLAG_ORIG_SACK_ACKED;
3430	} else {
3431	int delta;
3432
3433	/* Non-retransmitted hole got filled? That's reordering */
3434	if (before(seq1: reord, seq2: prior_fack))
3435	tcp_check_sack_reordering(sk, low_seq: reord, ts: 0);
3436
3437	delta = prior_sacked - tp->sacked_out;
3438	tp->lost_cnt_hint -= min(tp->lost_cnt_hint, delta);
3439	}
3440	} else if (skb && rtt_update && sack_rtt_us >= 0 &&
3441	sack_rtt_us > tcp_stamp_us_delta(t1: tp->tcp_mstamp,
3442	t0: tcp_skb_timestamp_us(skb))) {
3443	/* Do not re-arm RTO if the sack RTT is measured from data sent
3444	* after when the head was last (re)transmitted. Otherwise the
3445	* timeout may continue to extend in loss recovery.
3446	*/
3447	flag |= FLAG_SET_XMIT_TIMER; /* set TLP or RTO timer */
3448	}
3449
3450	if (icsk->icsk_ca_ops->pkts_acked) {
3451	struct ack_sample sample = { .pkts_acked = pkts_acked,
3452	.rtt_us = sack->rate->rtt_us };
3453
3454	sample.in_flight = tp->mss_cache *
3455	(tp->delivered - sack->rate->prior_delivered);
3456	icsk->icsk_ca_ops->pkts_acked(sk, &sample);
3457	}
3458
3459	#if FASTRETRANS_DEBUG > 0
3460	WARN_ON((int)tp->sacked_out < 0);
3461	WARN_ON((int)tp->lost_out < 0);
3462	WARN_ON((int)tp->retrans_out < 0);
3463	if (!tp->packets_out && tcp_is_sack(tp)) {
3464	icsk = inet_csk(sk);
3465	if (tp->lost_out) {
3466	pr_debug("Leak l=%u %d\n",
3467	tp->lost_out, icsk->icsk_ca_state);
3468	tp->lost_out = 0;
3469	}
3470	if (tp->sacked_out) {
3471	pr_debug("Leak s=%u %d\n",
3472	tp->sacked_out, icsk->icsk_ca_state);
3473	tp->sacked_out = 0;
3474	}
3475	if (tp->retrans_out) {
3476	pr_debug("Leak r=%u %d\n",
3477	tp->retrans_out, icsk->icsk_ca_state);
3478	tp->retrans_out = 0;
3479	}
3480	}
3481	#endif
3482	return flag;
3483	}
3484
3485	static void tcp_ack_probe(struct sock *sk)
3486	{
3487	struct inet_connection_sock *icsk = inet_csk(sk);
3488	struct sk_buff *head = tcp_send_head(sk);
3489	const struct tcp_sock *tp = tcp_sk(sk);
3490
3491	/* Was it a usable window open? */
3492	if (!head)
3493	return;
3494	if (!after(TCP_SKB_CB(head)->end_seq, tcp_wnd_end(tp))) {
3495	icsk->icsk_backoff = 0;
3496	icsk->icsk_probes_tstamp = 0;
3497	inet_csk_clear_xmit_timer(sk, ICSK_TIME_PROBE0);
3498	/* Socket must be waked up by subsequent tcp_data_snd_check().
3499	* This function is not for random using!
3500	*/
3501	} else {
3502	unsigned long when = tcp_probe0_when(sk, TCP_RTO_MAX);
3503
3504	when = tcp_clamp_probe0_to_user_timeout(sk, when);
3505	tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, when, TCP_RTO_MAX);
3506	}
3507	}
3508
3509	static inline bool tcp_ack_is_dubious(const struct sock *sk, const int flag)
3510	{
3511	return !(flag & FLAG_NOT_DUP) || (flag & FLAG_CA_ALERT) ||
3512	inet_csk(sk)->icsk_ca_state != TCP_CA_Open;
3513	}
3514
3515	/* Decide wheather to run the increase function of congestion control. */
3516	static inline bool tcp_may_raise_cwnd(const struct sock *sk, const int flag)
3517	{
3518	/* If reordering is high then always grow cwnd whenever data is
3519	* delivered regardless of its ordering. Otherwise stay conservative
3520	* and only grow cwnd on in-order delivery (RFC5681). A stretched ACK w/
3521	* new SACK or ECE mark may first advance cwnd here and later reduce
3522	* cwnd in tcp_fastretrans_alert() based on more states.
3523	*/
3524	if (tcp_sk(sk)->reordering >
3525	READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering))
3526	return flag & FLAG_FORWARD_PROGRESS;
3527
3528	return flag & FLAG_DATA_ACKED;
3529	}
3530
3531	/* The "ultimate" congestion control function that aims to replace the rigid
3532	* cwnd increase and decrease control (tcp_cong_avoid,tcp_*cwnd_reduction).
3533	* It's called toward the end of processing an ACK with precise rate
3534	* information. All transmission or retransmission are delayed afterwards.
3535	*/
3536	static void tcp_cong_control(struct sock *sk, u32 ack, u32 acked_sacked,
3537	int flag, const struct rate_sample *rs)
3538	{
3539	const struct inet_connection_sock *icsk = inet_csk(sk);
3540
3541	if (icsk->icsk_ca_ops->cong_control) {
3542	icsk->icsk_ca_ops->cong_control(sk, rs);
3543	return;
3544	}
3545
3546	if (tcp_in_cwnd_reduction(sk)) {
3547	/* Reduce cwnd if state mandates */
3548	tcp_cwnd_reduction(sk, newly_acked_sacked: acked_sacked, newly_lost: rs->losses, flag);
3549	} else if (tcp_may_raise_cwnd(sk, flag)) {
3550	/* Advance cwnd if state allows */
3551	tcp_cong_avoid(sk, ack, acked: acked_sacked);
3552	}
3553	tcp_update_pacing_rate(sk);
3554	}
3555
3556	/* Check that window update is acceptable.
3557	* The function assumes that snd_una<=ack<=snd_next.
3558	*/
3559	static inline bool tcp_may_update_window(const struct tcp_sock *tp,
3560	const u32 ack, const u32 ack_seq,
3561	const u32 nwin)
3562	{
3563	return after(ack, tp->snd_una) ||
3564	after(ack_seq, tp->snd_wl1) ||
3565	(ack_seq == tp->snd_wl1 && (nwin > tp->snd_wnd || !nwin));
3566	}
3567
3568	static void tcp_snd_sne_update(struct tcp_sock *tp, u32 ack)
3569	{
3570	#ifdef CONFIG_TCP_AO
3571	struct tcp_ao_info *ao;
3572
3573	if (!static_branch_unlikely(&tcp_ao_needed.key))
3574	return;
3575
3576	ao = rcu_dereference_protected(tp->ao_info,
3577	lockdep_sock_is_held((struct sock *)tp));
3578	if (ao && ack < tp->snd_una)
3579	ao->snd_sne++;
3580	#endif
3581	}
3582
3583	/* If we update tp->snd_una, also update tp->bytes_acked */
3584	static void tcp_snd_una_update(struct tcp_sock *tp, u32 ack)
3585	{
3586	u32 delta = ack - tp->snd_una;
3587
3588	sock_owned_by_me(sk: (struct sock *)tp);
3589	tp->bytes_acked += delta;
3590	tcp_snd_sne_update(tp, ack);
3591	tp->snd_una = ack;
3592	}
3593
3594	static void tcp_rcv_sne_update(struct tcp_sock *tp, u32 seq)
3595	{
3596	#ifdef CONFIG_TCP_AO
3597	struct tcp_ao_info *ao;
3598
3599	if (!static_branch_unlikely(&tcp_ao_needed.key))
3600	return;
3601
3602	ao = rcu_dereference_protected(tp->ao_info,
3603	lockdep_sock_is_held((struct sock *)tp));
3604	if (ao && seq < tp->rcv_nxt)
3605	ao->rcv_sne++;
3606	#endif
3607	}
3608
3609	/* If we update tp->rcv_nxt, also update tp->bytes_received */
3610	static void tcp_rcv_nxt_update(struct tcp_sock *tp, u32 seq)
3611	{
3612	u32 delta = seq - tp->rcv_nxt;
3613
3614	sock_owned_by_me(sk: (struct sock *)tp);
3615	tp->bytes_received += delta;
3616	tcp_rcv_sne_update(tp, seq);
3617	WRITE_ONCE(tp->rcv_nxt, seq);
3618	}
3619
3620	/* Update our send window.
3621	*
3622	* Window update algorithm, described in RFC793/RFC1122 (used in linux-2.2
3623	* and in FreeBSD. NetBSD's one is even worse.) is wrong.
3624	*/
3625	static int tcp_ack_update_window(struct sock *sk, const struct sk_buff *skb, u32 ack,
3626	u32 ack_seq)
3627	{
3628	struct tcp_sock *tp = tcp_sk(sk);
3629	int flag = 0;
3630	u32 nwin = ntohs(tcp_hdr(skb)->window);
3631
3632	if (likely(!tcp_hdr(skb)->syn))
3633	nwin <<= tp->rx_opt.snd_wscale;
3634
3635	if (tcp_may_update_window(tp, ack, ack_seq, nwin)) {
3636	flag |= FLAG_WIN_UPDATE;
3637	tcp_update_wl(tp, seq: ack_seq);
3638
3639	if (tp->snd_wnd != nwin) {
3640	tp->snd_wnd = nwin;
3641
3642	/* Note, it is the only place, where
3643	* fast path is recovered for sending TCP.
3644	*/
3645	tp->pred_flags = 0;
3646	tcp_fast_path_check(sk);
3647
3648	if (!tcp_write_queue_empty(sk))
3649	tcp_slow_start_after_idle_check(sk);
3650
3651	if (nwin > tp->max_window) {
3652	tp->max_window = nwin;
3653	tcp_sync_mss(sk, pmtu: inet_csk(sk)->icsk_pmtu_cookie);
3654	}
3655	}
3656	}
3657
3658	tcp_snd_una_update(tp, ack);
3659
3660	return flag;
3661	}
3662
3663	static bool __tcp_oow_rate_limited(struct net *net, int mib_idx,
3664	u32 *last_oow_ack_time)
3665	{
3666	/* Paired with the WRITE_ONCE() in this function. */
3667	u32 val = READ_ONCE(*last_oow_ack_time);
3668
3669	if (val) {
3670	s32 elapsed = (s32)(tcp_jiffies32 - val);
3671
3672	if (0 <= elapsed &&
3673	elapsed < READ_ONCE(net->ipv4.sysctl_tcp_invalid_ratelimit)) {
3674	NET_INC_STATS(net, mib_idx);
3675	return true; /* rate-limited: don't send yet! */
3676	}
3677	}
3678
3679	/* Paired with the prior READ_ONCE() and with itself,
3680	* as we might be lockless.
3681	*/
3682	WRITE_ONCE(*last_oow_ack_time, tcp_jiffies32);
3683
3684	return false; /* not rate-limited: go ahead, send dupack now! */
3685	}
3686
3687	/* Return true if we're currently rate-limiting out-of-window ACKs and
3688	* thus shouldn't send a dupack right now. We rate-limit dupacks in
3689	* response to out-of-window SYNs or ACKs to mitigate ACK loops or DoS
3690	* attacks that send repeated SYNs or ACKs for the same connection. To
3691	* do this, we do not send a duplicate SYNACK or ACK if the remote
3692	* endpoint is sending out-of-window SYNs or pure ACKs at a high rate.
3693	*/
3694	bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
3695	int mib_idx, u32 *last_oow_ack_time)
3696	{
3697	/* Data packets without SYNs are not likely part of an ACK loop. */
3698	if ((TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq) &&
3699	!tcp_hdr(skb)->syn)
3700	return false;
3701
3702	return __tcp_oow_rate_limited(net, mib_idx, last_oow_ack_time);
3703	}
3704
3705	/* RFC 5961 7 [ACK Throttling] */
3706	static void tcp_send_challenge_ack(struct sock *sk)
3707	{
3708	struct tcp_sock *tp = tcp_sk(sk);
3709	struct net *net = sock_net(sk);
3710	u32 count, now, ack_limit;
3711
3712	/* First check our per-socket dupack rate limit. */
3713	if (__tcp_oow_rate_limited(net,
3714	mib_idx: LINUX_MIB_TCPACKSKIPPEDCHALLENGE,
3715	last_oow_ack_time: &tp->last_oow_ack_time))
3716	return;
3717
3718	ack_limit = READ_ONCE(net->ipv4.sysctl_tcp_challenge_ack_limit);
3719	if (ack_limit == INT_MAX)
3720	goto send_ack;
3721
3722	/* Then check host-wide RFC 5961 rate limit. */
3723	now = jiffies / HZ;
3724	if (now != READ_ONCE(net->ipv4.tcp_challenge_timestamp)) {
3725	u32 half = (ack_limit + 1) >> 1;
3726
3727	WRITE_ONCE(net->ipv4.tcp_challenge_timestamp, now);
3728	WRITE_ONCE(net->ipv4.tcp_challenge_count,
3729	get_random_u32_inclusive(half, ack_limit + half - 1));
3730	}
3731	count = READ_ONCE(net->ipv4.tcp_challenge_count);
3732	if (count > 0) {
3733	WRITE_ONCE(net->ipv4.tcp_challenge_count, count - 1);
3734	send_ack:
3735	NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK);
3736	tcp_send_ack(sk);
3737	}
3738	}
3739
3740	static void tcp_store_ts_recent(struct tcp_sock *tp)
3741	{
3742	tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval;
3743	tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
3744	}
3745
3746	static void tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq)
3747	{
3748	if (tp->rx_opt.saw_tstamp && !after(seq, tp->rcv_wup)) {
3749	/* PAWS bug workaround wrt. ACK frames, the PAWS discard
3750	* extra check below makes sure this can only happen
3751	* for pure ACK frames. -DaveM
3752	*
3753	* Not only, also it occurs for expired timestamps.
3754	*/
3755
3756	if (tcp_paws_check(rx_opt: &tp->rx_opt, paws_win: 0))
3757	tcp_store_ts_recent(tp);
3758	}
3759	}
3760
3761	/* This routine deals with acks during a TLP episode and ends an episode by
3762	* resetting tlp_high_seq. Ref: TLP algorithm in draft-ietf-tcpm-rack
3763	*/
3764	static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag)
3765	{
3766	struct tcp_sock *tp = tcp_sk(sk);
3767
3768	if (before(seq1: ack, seq2: tp->tlp_high_seq))
3769	return;
3770
3771	if (!tp->tlp_retrans) {
3772	/* TLP of new data has been acknowledged */
3773	tp->tlp_high_seq = 0;
3774	} else if (flag & FLAG_DSACK_TLP) {
3775	/* This DSACK means original and TLP probe arrived; no loss */
3776	tp->tlp_high_seq = 0;
3777	} else if (after(ack, tp->tlp_high_seq)) {
3778	/* ACK advances: there was a loss, so reduce cwnd. Reset
3779	* tlp_high_seq in tcp_init_cwnd_reduction()
3780	*/
3781	tcp_init_cwnd_reduction(sk);
3782	tcp_set_ca_state(sk, ca_state: TCP_CA_CWR);
3783	tcp_end_cwnd_reduction(sk);
3784	tcp_try_keep_open(sk);
3785	NET_INC_STATS(sock_net(sk),
3786	LINUX_MIB_TCPLOSSPROBERECOVERY);
3787	} else if (!(flag & (FLAG_SND_UNA_ADVANCED |
3788	FLAG_NOT_DUP | FLAG_DATA_SACKED))) {
3789	/* Pure dupack: original and TLP probe arrived; no loss */
3790	tp->tlp_high_seq = 0;
3791	}
3792	}
3793
3794	static inline void tcp_in_ack_event(struct sock *sk, u32 flags)
3795	{
3796	const struct inet_connection_sock *icsk = inet_csk(sk);
3797
3798	if (icsk->icsk_ca_ops->in_ack_event)
3799	icsk->icsk_ca_ops->in_ack_event(sk, flags);
3800	}
3801
3802	/* Congestion control has updated the cwnd already. So if we're in
3803	* loss recovery then now we do any new sends (for FRTO) or
3804	* retransmits (for CA_Loss or CA_recovery) that make sense.
3805	*/
3806	static void tcp_xmit_recovery(struct sock *sk, int rexmit)
3807	{
3808	struct tcp_sock *tp = tcp_sk(sk);
3809
3810	if (rexmit == REXMIT_NONE || sk->sk_state == TCP_SYN_SENT)
3811	return;
3812
3813	if (unlikely(rexmit == REXMIT_NEW)) {
3814	__tcp_push_pending_frames(sk, cur_mss: tcp_current_mss(sk),
3815	TCP_NAGLE_OFF);
3816	if (after(tp->snd_nxt, tp->high_seq))
3817	return;
3818	tp->frto = 0;
3819	}
3820	tcp_xmit_retransmit_queue(sk);
3821	}
3822
3823	/* Returns the number of packets newly acked or sacked by the current ACK */
3824	static u32 tcp_newly_delivered(struct sock *sk, u32 prior_delivered, int flag)
3825	{
3826	const struct net *net = sock_net(sk);
3827	struct tcp_sock *tp = tcp_sk(sk);
3828	u32 delivered;
3829
3830	delivered = tp->delivered - prior_delivered;
3831	NET_ADD_STATS(net, LINUX_MIB_TCPDELIVERED, delivered);
3832	if (flag & FLAG_ECE)
3833	NET_ADD_STATS(net, LINUX_MIB_TCPDELIVEREDCE, delivered);
3834
3835	return delivered;
3836	}
3837
3838	/* This routine deals with incoming acks, but not outgoing ones. */
3839	static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
3840	{
3841	struct inet_connection_sock *icsk = inet_csk(sk);
3842	struct tcp_sock *tp = tcp_sk(sk);
3843	struct tcp_sacktag_state sack_state;
3844	struct rate_sample rs = { .prior_delivered = 0 };
3845	u32 prior_snd_una = tp->snd_una;
3846	bool is_sack_reneg = tp->is_sack_reneg;
3847	u32 ack_seq = TCP_SKB_CB(skb)->seq;
3848	u32 ack = TCP_SKB_CB(skb)->ack_seq;
3849	int num_dupack = 0;
3850	int prior_packets = tp->packets_out;
3851	u32 delivered = tp->delivered;
3852	u32 lost = tp->lost;
3853	int rexmit = REXMIT_NONE; /* Flag to (re)transmit to recover losses */
3854	u32 prior_fack;
3855
3856	sack_state.first_sackt = 0;
3857	sack_state.rate = &rs;
3858	sack_state.sack_delivered = 0;
3859
3860	/* We very likely will need to access rtx queue. */
3861	prefetch(sk->tcp_rtx_queue.rb_node);
3862
3863	/* If the ack is older than previous acks
3864	* then we can probably ignore it.
3865	*/
3866	if (before(seq1: ack, seq2: prior_snd_una)) {
3867	u32 max_window;
3868
3869	/* do not accept ACK for bytes we never sent. */
3870	max_window = min_t(u64, tp->max_window, tp->bytes_acked);
3871	/* RFC 5961 5.2 [Blind Data Injection Attack].[Mitigation] */
3872	if (before(seq1: ack, seq2: prior_snd_una - max_window)) {
3873	if (!(flag & FLAG_NO_CHALLENGE_ACK))
3874	tcp_send_challenge_ack(sk);
3875	return -SKB_DROP_REASON_TCP_TOO_OLD_ACK;
3876	}
3877	goto old_ack;
3878	}
3879
3880	/* If the ack includes data we haven't sent yet, discard
3881	* this segment (RFC793 Section 3.9).
3882	*/
3883	if (after(ack, tp->snd_nxt))
3884	return -SKB_DROP_REASON_TCP_ACK_UNSENT_DATA;
3885
3886	if (after(ack, prior_snd_una)) {
3887	flag |= FLAG_SND_UNA_ADVANCED;
3888	icsk->icsk_retransmits = 0;
3889
3890	#if IS_ENABLED(CONFIG_TLS_DEVICE)
3891	if (static_branch_unlikely(&clean_acked_data_enabled.key))
3892	if (icsk->icsk_clean_acked)
3893	icsk->icsk_clean_acked(sk, ack);
3894	#endif
3895	}
3896
3897	prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una;
3898	rs.prior_in_flight = tcp_packets_in_flight(tp);
3899
3900	/* ts_recent update must be made after we are sure that the packet
3901	* is in window.
3902	*/
3903	if (flag & FLAG_UPDATE_TS_RECENT)
3904	tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq);
3905
3906	if ((flag & (FLAG_SLOWPATH | FLAG_SND_UNA_ADVANCED)) ==
3907	FLAG_SND_UNA_ADVANCED) {
3908	/* Window is constant, pure forward advance.
3909	* No more checks are required.
3910	* Note, we use the fact that SND.UNA>=SND.WL2.
3911	*/
3912	tcp_update_wl(tp, seq: ack_seq);
3913	tcp_snd_una_update(tp, ack);
3914	flag |= FLAG_WIN_UPDATE;
3915
3916	tcp_in_ack_event(sk, flags: CA_ACK_WIN_UPDATE);
3917
3918	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPACKS);
3919	} else {
3920	u32 ack_ev_flags = CA_ACK_SLOWPATH;
3921
3922	if (ack_seq != TCP_SKB_CB(skb)->end_seq)
3923	flag |= FLAG_DATA;
3924	else
3925	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPUREACKS);
3926
3927	flag |= tcp_ack_update_window(sk, skb, ack, ack_seq);
3928
3929	if (TCP_SKB_CB(skb)->sacked)
3930	flag |= tcp_sacktag_write_queue(sk, ack_skb: skb, prior_snd_una,
3931	state: &sack_state);
3932
3933	if (tcp_ecn_rcv_ecn_echo(tp, th: tcp_hdr(skb))) {
3934	flag |= FLAG_ECE;
3935	ack_ev_flags |= CA_ACK_ECE;
3936	}
3937
3938	if (sack_state.sack_delivered)
3939	tcp_count_delivered(tp, delivered: sack_state.sack_delivered,
3940	ece_ack: flag & FLAG_ECE);
3941
3942	if (flag & FLAG_WIN_UPDATE)
3943	ack_ev_flags |= CA_ACK_WIN_UPDATE;
3944
3945	tcp_in_ack_event(sk, flags: ack_ev_flags);
3946	}
3947
3948	/* This is a deviation from RFC3168 since it states that:
3949	* "When the TCP data sender is ready to set the CWR bit after reducing
3950	* the congestion window, it SHOULD set the CWR bit only on the first
3951	* new data packet that it transmits."
3952	* We accept CWR on pure ACKs to be more robust
3953	* with widely-deployed TCP implementations that do this.
3954	*/
3955	tcp_ecn_accept_cwr(sk, skb);
3956
3957	/* We passed data and got it acked, remove any soft error
3958	* log. Something worked...
3959	*/
3960	WRITE_ONCE(sk->sk_err_soft, 0);
3961	icsk->icsk_probes_out = 0;
3962	tp->rcv_tstamp = tcp_jiffies32;
3963	if (!prior_packets)
3964	goto no_queue;
3965
3966	/* See if we can take anything off of the retransmit queue. */
3967	flag |= tcp_clean_rtx_queue(sk, ack_skb: skb, prior_fack, prior_snd_una,
3968	sack: &sack_state, ece_ack: flag & FLAG_ECE);
3969
3970	tcp_rack_update_reo_wnd(sk, rs: &rs);
3971
3972	if (tp->tlp_high_seq)
3973	tcp_process_tlp_ack(sk, ack, flag);
3974
3975	if (tcp_ack_is_dubious(sk, flag)) {
3976	if (!(flag & (FLAG_SND_UNA_ADVANCED |
3977	FLAG_NOT_DUP | FLAG_DSACKING_ACK))) {
3978	num_dupack = 1;
3979	/* Consider if pure acks were aggregated in tcp_add_backlog() */
3980	if (!(flag & FLAG_DATA))
3981	num_dupack = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
3982	}
3983	tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, ack_flag: &flag,
3984	rexmit: &rexmit);
3985	}
3986
3987	/* If needed, reset TLP/RTO timer when RACK doesn't set. */
3988	if (flag & FLAG_SET_XMIT_TIMER)
3989	tcp_set_xmit_timer(sk);
3990
3991	if ((flag & FLAG_FORWARD_PROGRESS) || !(flag & FLAG_NOT_DUP))
3992	sk_dst_confirm(sk);
3993
3994	delivered = tcp_newly_delivered(sk, prior_delivered: delivered, flag);
3995	lost = tp->lost - lost; /* freshly marked lost */
3996	rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED);
3997	tcp_rate_gen(sk, delivered, lost, is_sack_reneg, rs: sack_state.rate);
3998	tcp_cong_control(sk, ack, acked_sacked: delivered, flag, rs: sack_state.rate);
3999	tcp_xmit_recovery(sk, rexmit);
4000	return 1;
4001
4002	no_queue:
4003	/* If data was DSACKed, see if we can undo a cwnd reduction. */
4004	if (flag & FLAG_DSACKING_ACK) {
4005	tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, ack_flag: &flag,
4006	rexmit: &rexmit);
4007	tcp_newly_delivered(sk, prior_delivered: delivered, flag);
4008	}
4009	/* If this ack opens up a zero window, clear backoff. It was
4010	* being used to time the probes, and is probably far higher than
4011	* it needs to be for normal retransmission.
4012	*/
4013	tcp_ack_probe(sk);
4014
4015	if (tp->tlp_high_seq)
4016	tcp_process_tlp_ack(sk, ack, flag);
4017	return 1;
4018
4019	old_ack:
4020	/* If data was SACKed, tag it and see if we should send more data.
4021	* If data was DSACKed, see if we can undo a cwnd reduction.
4022	*/
4023	if (TCP_SKB_CB(skb)->sacked) {
4024	flag |= tcp_sacktag_write_queue(sk, ack_skb: skb, prior_snd_una,
4025	state: &sack_state);
4026	tcp_fastretrans_alert(sk, prior_snd_una, num_dupack, ack_flag: &flag,
4027	rexmit: &rexmit);
4028	tcp_newly_delivered(sk, prior_delivered: delivered, flag);
4029	tcp_xmit_recovery(sk, rexmit);
4030	}
4031
4032	return 0;
4033	}
4034
4035	static void tcp_parse_fastopen_option(int len, const unsigned char *cookie,
4036	bool syn, struct tcp_fastopen_cookie *foc,
4037	bool exp_opt)
4038	{
4039	/* Valid only in SYN or SYN-ACK with an even length. */
4040	if (!foc || !syn || len < 0 || (len & 1))
4041	return;
4042
4043	if (len >= TCP_FASTOPEN_COOKIE_MIN &&
4044	len <= TCP_FASTOPEN_COOKIE_MAX)
4045	memcpy(foc->val, cookie, len);
4046	else if (len != 0)
4047	len = -1;
4048	foc->len = len;
4049	foc->exp = exp_opt;
4050	}
4051
4052	static bool smc_parse_options(const struct tcphdr *th,
4053	struct tcp_options_received *opt_rx,
4054	const unsigned char *ptr,
4055	int opsize)
4056	{
4057	#if IS_ENABLED(CONFIG_SMC)
4058	if (static_branch_unlikely(&tcp_have_smc)) {
4059	if (th->syn && !(opsize & 1) &&
4060	opsize >= TCPOLEN_EXP_SMC_BASE &&
4061	get_unaligned_be32(p: ptr) == TCPOPT_SMC_MAGIC) {
4062	opt_rx->smc_ok = 1;
4063	return true;
4064	}
4065	}
4066	#endif
4067	return false;
4068	}
4069
4070	/* Try to parse the MSS option from the TCP header. Return 0 on failure, clamped
4071	* value on success.
4072	*/
4073	u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss)
4074	{
4075	const unsigned char *ptr = (const unsigned char *)(th + 1);
4076	int length = (th->doff * 4) - sizeof(struct tcphdr);
4077	u16 mss = 0;
4078
4079	while (length > 0) {
4080	int opcode = *ptr++;
4081	int opsize;
4082
4083	switch (opcode) {
4084	case TCPOPT_EOL:
4085	return mss;
4086	case TCPOPT_NOP: /* Ref: RFC 793 section 3.1 */
4087	length--;
4088	continue;
4089	default:
4090	if (length < 2)
4091	return mss;
4092	opsize = *ptr++;
4093	if (opsize < 2) /* "silly options" */
4094	return mss;
4095	if (opsize > length)
4096	return mss; /* fail on partial options */
4097	if (opcode == TCPOPT_MSS && opsize == TCPOLEN_MSS) {
4098	u16 in_mss = get_unaligned_be16(p: ptr);
4099
4100	if (in_mss) {
4101	if (user_mss && user_mss < in_mss)
4102	in_mss = user_mss;
4103	mss = in_mss;
4104	}
4105	}
4106	ptr += opsize - 2;
4107	length -= opsize;
4108	}
4109	}
4110	return mss;
4111	}
4112	EXPORT_SYMBOL_GPL(tcp_parse_mss_option);
4113
4114	/* Look for tcp options. Normally only called on SYN and SYNACK packets.
4115	* But, this can also be called on packets in the established flow when
4116	* the fast version below fails.
4117	*/
4118	void tcp_parse_options(const struct net *net,
4119	const struct sk_buff *skb,
4120	struct tcp_options_received *opt_rx, int estab,
4121	struct tcp_fastopen_cookie *foc)
4122	{
4123	const unsigned char *ptr;
4124	const struct tcphdr *th = tcp_hdr(skb);
4125	int length = (th->doff * 4) - sizeof(struct tcphdr);
4126
4127	ptr = (const unsigned char *)(th + 1);
4128	opt_rx->saw_tstamp = 0;
4129	opt_rx->saw_unknown = 0;
4130
4131	while (length > 0) {
4132	int opcode = *ptr++;
4133	int opsize;
4134
4135	switch (opcode) {
4136	case TCPOPT_EOL:
4137	return;
4138	case TCPOPT_NOP: /* Ref: RFC 793 section 3.1 */
4139	length--;
4140	continue;
4141	default:
4142	if (length < 2)
4143	return;
4144	opsize = *ptr++;
4145	if (opsize < 2) /* "silly options" */
4146	return;
4147	if (opsize > length)
4148	return; /* don't parse partial options */
4149	switch (opcode) {
4150	case TCPOPT_MSS:
4151	if (opsize == TCPOLEN_MSS && th->syn && !estab) {
4152	u16 in_mss = get_unaligned_be16(p: ptr);
4153	if (in_mss) {
4154	if (opt_rx->user_mss &&
4155	opt_rx->user_mss < in_mss)
4156	in_mss = opt_rx->user_mss;
4157	opt_rx->mss_clamp = in_mss;
4158	}
4159	}
4160	break;
4161	case TCPOPT_WINDOW:
4162	if (opsize == TCPOLEN_WINDOW && th->syn &&
4163	!estab && READ_ONCE(net->ipv4.sysctl_tcp_window_scaling)) {
4164	__u8 snd_wscale = *(__u8 *)ptr;
4165	opt_rx->wscale_ok = 1;
4166	if (snd_wscale > TCP_MAX_WSCALE) {
4167	net_info_ratelimited("%s: Illegal window scaling value %d > %u received\n",
4168	__func__,
4169	snd_wscale,
4170	TCP_MAX_WSCALE);
4171	snd_wscale = TCP_MAX_WSCALE;
4172	}
4173	opt_rx->snd_wscale = snd_wscale;
4174	}
4175	break;
4176	case TCPOPT_TIMESTAMP:
4177	if ((opsize == TCPOLEN_TIMESTAMP) &&
4178	((estab && opt_rx->tstamp_ok) ||
4179	(!estab && READ_ONCE(net->ipv4.sysctl_tcp_timestamps)))) {
4180	opt_rx->saw_tstamp = 1;
4181	opt_rx->rcv_tsval = get_unaligned_be32(p: ptr);
4182	opt_rx->rcv_tsecr = get_unaligned_be32(p: ptr + 4);
4183	}
4184	break;
4185	case TCPOPT_SACK_PERM:
4186	if (opsize == TCPOLEN_SACK_PERM && th->syn &&
4187	!estab && READ_ONCE(net->ipv4.sysctl_tcp_sack)) {
4188	opt_rx->sack_ok = TCP_SACK_SEEN;
4189	tcp_sack_reset(rx_opt: opt_rx);
4190	}
4191	break;
4192
4193	case TCPOPT_SACK:
4194	if ((opsize >= (TCPOLEN_SACK_BASE + TCPOLEN_SACK_PERBLOCK)) &&
4195	!((opsize - TCPOLEN_SACK_BASE) % TCPOLEN_SACK_PERBLOCK) &&
4196	opt_rx->sack_ok) {
4197	TCP_SKB_CB(skb)->sacked = (ptr - 2) - (unsigned char *)th;
4198	}
4199	break;
4200	#ifdef CONFIG_TCP_MD5SIG
4201	case TCPOPT_MD5SIG:
4202	/* The MD5 Hash has already been
4203	* checked (see tcp_v{4,6}_rcv()).
4204	*/
4205	break;
4206	#endif
4207	case TCPOPT_FASTOPEN:
4208	tcp_parse_fastopen_option(
4209	len: opsize - TCPOLEN_FASTOPEN_BASE,
4210	cookie: ptr, syn: th->syn, foc, exp_opt: false);
4211	break;
4212
4213	case TCPOPT_EXP:
4214	/* Fast Open option shares code 254 using a
4215	* 16 bits magic number.
4216	*/
4217	if (opsize >= TCPOLEN_EXP_FASTOPEN_BASE &&
4218	get_unaligned_be16(p: ptr) ==
4219	TCPOPT_FASTOPEN_MAGIC) {
4220	tcp_parse_fastopen_option(len: opsize -
4221	TCPOLEN_EXP_FASTOPEN_BASE,
4222	cookie: ptr + 2, syn: th->syn, foc, exp_opt: true);
4223	break;
4224	}
4225
4226	if (smc_parse_options(th, opt_rx, ptr, opsize))
4227	break;
4228
4229	opt_rx->saw_unknown = 1;
4230	break;
4231
4232	default:
4233	opt_rx->saw_unknown = 1;
4234	}
4235	ptr += opsize-2;
4236	length -= opsize;
4237	}
4238	}
4239	}
4240	EXPORT_SYMBOL(tcp_parse_options);
4241
4242	static bool tcp_parse_aligned_timestamp(struct tcp_sock *tp, const struct tcphdr *th)
4243	{
4244	const __be32 *ptr = (const __be32 *)(th + 1);
4245
4246	if (*ptr == htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16)
4247	| (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) {
4248	tp->rx_opt.saw_tstamp = 1;
4249	++ptr;
4250	tp->rx_opt.rcv_tsval = ntohl(*ptr);
4251	++ptr;
4252	if (*ptr)
4253	tp->rx_opt.rcv_tsecr = ntohl(*ptr) - tp->tsoffset;
4254	else
4255	tp->rx_opt.rcv_tsecr = 0;
4256	return true;
4257	}
4258	return false;
4259	}
4260
4261	/* Fast parse options. This hopes to only see timestamps.
4262	* If it is wrong it falls back on tcp_parse_options().
4263	*/
4264	static bool tcp_fast_parse_options(const struct net *net,
4265	const struct sk_buff *skb,
4266	const struct tcphdr *th, struct tcp_sock *tp)
4267	{
4268	/* In the spirit of fast parsing, compare doff directly to constant
4269	* values. Because equality is used, short doff can be ignored here.
4270	*/
4271	if (th->doff == (sizeof(*th) / 4)) {
4272	tp->rx_opt.saw_tstamp = 0;
4273	return false;
4274	} else if (tp->rx_opt.tstamp_ok &&
4275	th->doff == ((sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4)) {
4276	if (tcp_parse_aligned_timestamp(tp, th))
4277	return true;
4278	}
4279
4280	tcp_parse_options(net, skb, &tp->rx_opt, 1, NULL);
4281	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
4282	tp->rx_opt.rcv_tsecr -= tp->tsoffset;
4283
4284	return true;
4285	}
4286
4287	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
4288	/*
4289	* Parse Signature options
4290	*/
4291	int tcp_do_parse_auth_options(const struct tcphdr *th,
4292	const u8 **md5_hash, const u8 **ao_hash)
4293	{
4294	int length = (th->doff << 2) - sizeof(*th);
4295	const u8 *ptr = (const u8 *)(th + 1);
4296	unsigned int minlen = TCPOLEN_MD5SIG;
4297
4298	if (IS_ENABLED(CONFIG_TCP_AO))
4299	minlen = sizeof(struct tcp_ao_hdr) + 1;
4300
4301	*md5_hash = NULL;
4302	*ao_hash = NULL;
4303
4304	/* If not enough data remaining, we can short cut */
4305	while (length >= minlen) {
4306	int opcode = *ptr++;
4307	int opsize;
4308
4309	switch (opcode) {
4310	case TCPOPT_EOL:
4311	return 0;
4312	case TCPOPT_NOP:
4313	length--;
4314	continue;
4315	default:
4316	opsize = *ptr++;
4317	if (opsize < 2 || opsize > length)
4318	return -EINVAL;
4319	if (opcode == TCPOPT_MD5SIG) {
4320	if (opsize != TCPOLEN_MD5SIG)
4321	return -EINVAL;
4322	if (unlikely(*md5_hash || *ao_hash))
4323	return -EEXIST;
4324	*md5_hash = ptr;
4325	} else if (opcode == TCPOPT_AO) {
4326	if (opsize <= sizeof(struct tcp_ao_hdr))
4327	return -EINVAL;
4328	if (unlikely(*md5_hash || *ao_hash))
4329	return -EEXIST;
4330	*ao_hash = ptr;
4331	}
4332	}
4333	ptr += opsize - 2;
4334	length -= opsize;
4335	}
4336	return 0;
4337	}
4338	EXPORT_SYMBOL(tcp_do_parse_auth_options);
4339	#endif
4340
4341	/* Sorry, PAWS as specified is broken wrt. pure-ACKs -DaveM
4342	*
4343	* It is not fatal. If this ACK does _not_ change critical state (seqs, window)
4344	* it can pass through stack. So, the following predicate verifies that
4345	* this segment is not used for anything but congestion avoidance or
4346	* fast retransmit. Moreover, we even are able to eliminate most of such
4347	* second order effects, if we apply some small "replay" window (~RTO)
4348	* to timestamp space.
4349	*
4350	* All these measures still do not guarantee that we reject wrapped ACKs
4351	* on networks with high bandwidth, when sequence space is recycled fastly,
4352	* but it guarantees that such events will be very rare and do not affect
4353	* connection seriously. This doesn't look nice, but alas, PAWS is really
4354	* buggy extension.
4355	*
4356	* [ Later note. Even worse! It is buggy for segments _with_ data. RFC
4357	* states that events when retransmit arrives after original data are rare.
4358	* It is a blatant lie. VJ forgot about fast retransmit! 8)8) It is
4359	* the biggest problem on large power networks even with minor reordering.
4360	* OK, let's give it small replay window. If peer clock is even 1hz, it is safe
4361	* up to bandwidth of 18Gigabit/sec. 8) ]
4362	*/
4363
4364	/* Estimates max number of increments of remote peer TSval in
4365	* a replay window (based on our current RTO estimation).
4366	*/
4367	static u32 tcp_tsval_replay(const struct sock *sk)
4368	{
4369	/* If we use usec TS resolution,
4370	* then expect the remote peer to use the same resolution.
4371	*/
4372	if (tcp_sk(sk)->tcp_usec_ts)
4373	return inet_csk(sk)->icsk_rto * (USEC_PER_SEC / HZ);
4374
4375	/* RFC 7323 recommends a TSval clock between 1ms and 1sec.
4376	* We know that some OS (including old linux) can use 1200 Hz.
4377	*/
4378	return inet_csk(sk)->icsk_rto * 1200 / HZ;
4379	}
4380
4381	static int tcp_disordered_ack(const struct sock *sk, const struct sk_buff *skb)
4382	{
4383	const struct tcp_sock *tp = tcp_sk(sk);
4384	const struct tcphdr *th = tcp_hdr(skb);
4385	u32 seq = TCP_SKB_CB(skb)->seq;
4386	u32 ack = TCP_SKB_CB(skb)->ack_seq;
4387
4388	return /* 1. Pure ACK with correct sequence number. */
4389	(th->ack && seq == TCP_SKB_CB(skb)->end_seq && seq == tp->rcv_nxt) &&
4390
4391	/* 2. ... and duplicate ACK. */
4392	ack == tp->snd_una &&
4393
4394	/* 3. ... and does not update window. */
4395	!tcp_may_update_window(tp, ack, ack_seq: seq, ntohs(th->window) << tp->rx_opt.snd_wscale) &&
4396
4397	/* 4. ... and sits in replay window. */
4398	(s32)(tp->rx_opt.ts_recent - tp->rx_opt.rcv_tsval) <=
4399	tcp_tsval_replay(sk);
4400	}
4401
4402	static inline bool tcp_paws_discard(const struct sock *sk,
4403	const struct sk_buff *skb)
4404	{
4405	const struct tcp_sock *tp = tcp_sk(sk);
4406
4407	return !tcp_paws_check(rx_opt: &tp->rx_opt, TCP_PAWS_WINDOW) &&
4408	!tcp_disordered_ack(sk, skb);
4409	}
4410
4411	/* Check segment sequence number for validity.
4412	*
4413	* Segment controls are considered valid, if the segment
4414	* fits to the window after truncation to the window. Acceptability
4415	* of data (and SYN, FIN, of course) is checked separately.
4416	* See tcp_data_queue(), for example.
4417	*
4418	* Also, controls (RST is main one) are accepted using RCV.WUP instead
4419	* of RCV.NXT. Peer still did not advance his SND.UNA when we
4420	* delayed ACK, so that hisSND.UNA<=ourRCV.WUP.
4421	* (borrowed from freebsd)
4422	*/
4423
4424	static enum skb_drop_reason tcp_sequence(const struct tcp_sock *tp,
4425	u32 seq, u32 end_seq)
4426	{
4427	if (before(seq1: end_seq, seq2: tp->rcv_wup))
4428	return SKB_DROP_REASON_TCP_OLD_SEQUENCE;
4429
4430	if (after(seq, tp->rcv_nxt + tcp_receive_window(tp)))
4431	return SKB_DROP_REASON_TCP_INVALID_SEQUENCE;
4432
4433	return SKB_NOT_DROPPED_YET;
4434	}
4435
4436	/* When we get a reset we do this. */
4437	void tcp_reset(struct sock *sk, struct sk_buff *skb)
4438	{
4439	trace_tcp_receive_reset(sk);
4440
4441	/* mptcp can't tell us to ignore reset pkts,
4442	* so just ignore the return value of mptcp_incoming_options().
4443	*/
4444	if (sk_is_mptcp(sk))
4445	mptcp_incoming_options(sk, skb);
4446
4447	/* We want the right error as BSD sees it (and indeed as we do). */
4448	switch (sk->sk_state) {
4449	case TCP_SYN_SENT:
4450	WRITE_ONCE(sk->sk_err, ECONNREFUSED);
4451	break;
4452	case TCP_CLOSE_WAIT:
4453	WRITE_ONCE(sk->sk_err, EPIPE);
4454	break;
4455	case TCP_CLOSE:
4456	return;
4457	default:
4458	WRITE_ONCE(sk->sk_err, ECONNRESET);
4459	}
4460	/* This barrier is coupled with smp_rmb() in tcp_poll() */
4461	smp_wmb();
4462
4463	tcp_write_queue_purge(sk);
4464	tcp_done(sk);
4465
4466	if (!sock_flag(sk, flag: SOCK_DEAD))
4467	sk_error_report(sk);
4468	}
4469
4470	/*
4471	* Process the FIN bit. This now behaves as it is supposed to work
4472	* and the FIN takes effect when it is validly part of sequence
4473	* space. Not before when we get holes.
4474	*
4475	* If we are ESTABLISHED, a received fin moves us to CLOSE-WAIT
4476	* (and thence onto LAST-ACK and finally, CLOSE, we never enter
4477	* TIME-WAIT)
4478	*
4479	* If we are in FINWAIT-1, a received FIN indicates simultaneous
4480	* close and we go into CLOSING (and later onto TIME-WAIT)
4481	*
4482	* If we are in FINWAIT-2, a received FIN moves us to TIME-WAIT.
4483	*/
4484	void tcp_fin(struct sock *sk)
4485	{
4486	struct tcp_sock *tp = tcp_sk(sk);
4487
4488	inet_csk_schedule_ack(sk);
4489
4490	WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | RCV_SHUTDOWN);
4491	sock_set_flag(sk, flag: SOCK_DONE);
4492
4493	switch (sk->sk_state) {
4494	case TCP_SYN_RECV:
4495	case TCP_ESTABLISHED:
4496	/* Move to CLOSE_WAIT */
4497	tcp_set_state(sk, state: TCP_CLOSE_WAIT);
4498	inet_csk_enter_pingpong_mode(sk);
4499	break;
4500
4501	case TCP_CLOSE_WAIT:
4502	case TCP_CLOSING:
4503	/* Received a retransmission of the FIN, do
4504	* nothing.
4505	*/
4506	break;
4507	case TCP_LAST_ACK:
4508	/* RFC793: Remain in the LAST-ACK state. */
4509	break;
4510
4511	case TCP_FIN_WAIT1:
4512	/* This case occurs when a simultaneous close
4513	* happens, we must ack the received FIN and
4514	* enter the CLOSING state.
4515	*/
4516	tcp_send_ack(sk);
4517	tcp_set_state(sk, state: TCP_CLOSING);
4518	break;
4519	case TCP_FIN_WAIT2:
4520	/* Received a FIN -- send ACK and enter TIME_WAIT. */
4521	tcp_send_ack(sk);
4522	tcp_time_wait(sk, state: TCP_TIME_WAIT, timeo: 0);
4523	break;
4524	default:
4525	/* Only TCP_LISTEN and TCP_CLOSE are left, in these
4526	* cases we should never reach this piece of code.
4527	*/
4528	pr_err("%s: Impossible, sk->sk_state=%d\n",
4529	__func__, sk->sk_state);
4530	break;
4531	}
4532
4533	/* It _is_ possible, that we have something out-of-order _after_ FIN.
4534	* Probably, we should reset in this case. For now drop them.
4535	*/
4536	skb_rbtree_purge(root: &tp->out_of_order_queue);
4537	if (tcp_is_sack(tp))
4538	tcp_sack_reset(rx_opt: &tp->rx_opt);
4539
4540	if (!sock_flag(sk, flag: SOCK_DEAD)) {
4541	sk->sk_state_change(sk);
4542
4543	/* Do not send POLL_HUP for half duplex close. */
4544	if (sk->sk_shutdown == SHUTDOWN_MASK ||
4545	sk->sk_state == TCP_CLOSE)
4546	sk_wake_async(sk, how: SOCK_WAKE_WAITD, POLL_HUP);
4547	else
4548	sk_wake_async(sk, how: SOCK_WAKE_WAITD, POLL_IN);
4549	}
4550	}
4551
4552	static inline bool tcp_sack_extend(struct tcp_sack_block *sp, u32 seq,
4553	u32 end_seq)
4554	{
4555	if (!after(seq, sp->end_seq) && !after(sp->start_seq, end_seq)) {
4556	if (before(seq1: seq, seq2: sp->start_seq))
4557	sp->start_seq = seq;
4558	if (after(end_seq, sp->end_seq))
4559	sp->end_seq = end_seq;
4560	return true;
4561	}
4562	return false;
4563	}
4564
4565	static void tcp_dsack_set(struct sock *sk, u32 seq, u32 end_seq)
4566	{
4567	struct tcp_sock *tp = tcp_sk(sk);
4568
4569	if (tcp_is_sack(tp) && READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_dsack)) {
4570	int mib_idx;
4571
4572	if (before(seq1: seq, seq2: tp->rcv_nxt))
4573	mib_idx = LINUX_MIB_TCPDSACKOLDSENT;
4574	else
4575	mib_idx = LINUX_MIB_TCPDSACKOFOSENT;
4576
4577	NET_INC_STATS(sock_net(sk), mib_idx);
4578
4579	tp->rx_opt.dsack = 1;
4580	tp->duplicate_sack[0].start_seq = seq;
4581	tp->duplicate_sack[0].end_seq = end_seq;
4582	}
4583	}
4584
4585	static void tcp_dsack_extend(struct sock *sk, u32 seq, u32 end_seq)
4586	{
4587	struct tcp_sock *tp = tcp_sk(sk);
4588
4589	if (!tp->rx_opt.dsack)
4590	tcp_dsack_set(sk, seq, end_seq);
4591	else
4592	tcp_sack_extend(sp: tp->duplicate_sack, seq, end_seq);
4593	}
4594
4595	static void tcp_rcv_spurious_retrans(struct sock *sk, const struct sk_buff *skb)
4596	{
4597	/* When the ACK path fails or drops most ACKs, the sender would
4598	* timeout and spuriously retransmit the same segment repeatedly.
4599	* If it seems our ACKs are not reaching the other side,
4600	* based on receiving a duplicate data segment with new flowlabel
4601	* (suggesting the sender suffered an RTO), and we are not already
4602	* repathing due to our own RTO, then rehash the socket to repath our
4603	* packets.
4604	*/
4605	#if IS_ENABLED(CONFIG_IPV6)
4606	if (inet_csk(sk)->icsk_ca_state != TCP_CA_Loss &&
4607	skb->protocol == htons(ETH_P_IPV6) &&
4608	(tcp_sk(sk)->inet_conn.icsk_ack.lrcv_flowlabel !=
4609	ntohl(ip6_flowlabel(ipv6_hdr(skb)))) &&
4610	sk_rethink_txhash(sk))
4611	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDUPLICATEDATAREHASH);
4612
4613	/* Save last flowlabel after a spurious retrans. */
4614	tcp_save_lrcv_flowlabel(sk, skb);
4615	#endif
4616	}
4617
4618	static void tcp_send_dupack(struct sock *sk, const struct sk_buff *skb)
4619	{
4620	struct tcp_sock *tp = tcp_sk(sk);
4621
4622	if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
4623	before(TCP_SKB_CB(skb)->seq, seq2: tp->rcv_nxt)) {
4624	NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
4625	tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
4626
4627	if (tcp_is_sack(tp) && READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_dsack)) {
4628	u32 end_seq = TCP_SKB_CB(skb)->end_seq;
4629
4630	tcp_rcv_spurious_retrans(sk, skb);
4631	if (after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))
4632	end_seq = tp->rcv_nxt;
4633	tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, end_seq);
4634	}
4635	}
4636
4637	tcp_send_ack(sk);
4638	}
4639
4640	/* These routines update the SACK block as out-of-order packets arrive or
4641	* in-order packets close up the sequence space.
4642	*/
4643	static void tcp_sack_maybe_coalesce(struct tcp_sock *tp)
4644	{
4645	int this_sack;
4646	struct tcp_sack_block *sp = &tp->selective_acks[0];
4647	struct tcp_sack_block *swalk = sp + 1;
4648
4649	/* See if the recent change to the first SACK eats into
4650	* or hits the sequence space of other SACK blocks, if so coalesce.
4651	*/
4652	for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;) {
4653	if (tcp_sack_extend(sp, seq: swalk->start_seq, end_seq: swalk->end_seq)) {
4654	int i;
4655
4656	/* Zap SWALK, by moving every further SACK up by one slot.
4657	* Decrease num_sacks.
4658	*/
4659	tp->rx_opt.num_sacks--;
4660	for (i = this_sack; i < tp->rx_opt.num_sacks; i++)
4661	sp[i] = sp[i + 1];
4662	continue;
4663	}
4664	this_sack++;
4665	swalk++;
4666	}
4667	}
4668
4669	void tcp_sack_compress_send_ack(struct sock *sk)
4670	{
4671	struct tcp_sock *tp = tcp_sk(sk);
4672
4673	if (!tp->compressed_ack)
4674	return;
4675
4676	if (hrtimer_try_to_cancel(timer: &tp->compressed_ack_timer) == 1)
4677	__sock_put(sk);
4678
4679	/* Since we have to send one ack finally,
4680	* substract one from tp->compressed_ack to keep
4681	* LINUX_MIB_TCPACKCOMPRESSED accurate.
4682	*/
4683	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
4684	tp->compressed_ack - 1);
4685
4686	tp->compressed_ack = 0;
4687	tcp_send_ack(sk);
4688	}
4689
4690	/* Reasonable amount of sack blocks included in TCP SACK option
4691	* The max is 4, but this becomes 3 if TCP timestamps are there.
4692	* Given that SACK packets might be lost, be conservative and use 2.
4693	*/
4694	#define TCP_SACK_BLOCKS_EXPECTED 2
4695
4696	static void tcp_sack_new_ofo_skb(struct sock *sk, u32 seq, u32 end_seq)
4697	{
4698	struct tcp_sock *tp = tcp_sk(sk);
4699	struct tcp_sack_block *sp = &tp->selective_acks[0];
4700	int cur_sacks = tp->rx_opt.num_sacks;
4701	int this_sack;
4702
4703	if (!cur_sacks)
4704	goto new_sack;
4705
4706	for (this_sack = 0; this_sack < cur_sacks; this_sack++, sp++) {
4707	if (tcp_sack_extend(sp, seq, end_seq)) {
4708	if (this_sack >= TCP_SACK_BLOCKS_EXPECTED)
4709	tcp_sack_compress_send_ack(sk);
4710	/* Rotate this_sack to the first one. */
4711	for (; this_sack > 0; this_sack--, sp--)
4712	swap(*sp, *(sp - 1));
4713	if (cur_sacks > 1)
4714	tcp_sack_maybe_coalesce(tp);
4715	return;
4716	}
4717	}
4718
4719	if (this_sack >= TCP_SACK_BLOCKS_EXPECTED)
4720	tcp_sack_compress_send_ack(sk);
4721
4722	/* Could not find an adjacent existing SACK, build a new one,
4723	* put it at the front, and shift everyone else down. We
4724	* always know there is at least one SACK present already here.
4725	*
4726	* If the sack array is full, forget about the last one.
4727	*/
4728	if (this_sack >= TCP_NUM_SACKS) {
4729	this_sack--;
4730	tp->rx_opt.num_sacks--;
4731	sp--;
4732	}
4733	for (; this_sack > 0; this_sack--, sp--)
4734	*sp = *(sp - 1);
4735
4736	new_sack:
4737	/* Build the new head SACK, and we're done. */
4738	sp->start_seq = seq;
4739	sp->end_seq = end_seq;
4740	tp->rx_opt.num_sacks++;
4741	}
4742
4743	/* RCV.NXT advances, some SACKs should be eaten. */
4744
4745	static void tcp_sack_remove(struct tcp_sock *tp)
4746	{
4747	struct tcp_sack_block *sp = &tp->selective_acks[0];
4748	int num_sacks = tp->rx_opt.num_sacks;
4749	int this_sack;
4750
4751	/* Empty ofo queue, hence, all the SACKs are eaten. Clear. */
4752	if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
4753	tp->rx_opt.num_sacks = 0;
4754	return;
4755	}
4756
4757	for (this_sack = 0; this_sack < num_sacks;) {
4758	/* Check if the start of the sack is covered by RCV.NXT. */
4759	if (!before(seq1: tp->rcv_nxt, seq2: sp->start_seq)) {
4760	int i;
4761
4762	/* RCV.NXT must cover all the block! */
4763	WARN_ON(before(tp->rcv_nxt, sp->end_seq));
4764
4765	/* Zap this SACK, by moving forward any other SACKS. */
4766	for (i = this_sack+1; i < num_sacks; i++)
4767	tp->selective_acks[i-1] = tp->selective_acks[i];
4768	num_sacks--;
4769	continue;
4770	}
4771	this_sack++;
4772	sp++;
4773	}
4774	tp->rx_opt.num_sacks = num_sacks;
4775	}
4776
4777	/**
4778	* tcp_try_coalesce - try to merge skb to prior one
4779	* @sk: socket
4780	* @to: prior buffer
4781	* @from: buffer to add in queue
4782	* @fragstolen: pointer to boolean
4783	*
4784	* Before queueing skb @from after @to, try to merge them
4785	* to reduce overall memory use and queue lengths, if cost is small.
4786	* Packets in ofo or receive queues can stay a long time.
4787	* Better try to coalesce them right now to avoid future collapses.
4788	* Returns true if caller should free @from instead of queueing it
4789	*/
4790	static bool tcp_try_coalesce(struct sock *sk,
4791	struct sk_buff *to,
4792	struct sk_buff *from,
4793	bool *fragstolen)
4794	{
4795	int delta;
4796
4797	*fragstolen = false;
4798
4799	/* Its possible this segment overlaps with prior segment in queue */
4800	if (TCP_SKB_CB(from)->seq != TCP_SKB_CB(to)->end_seq)
4801	return false;
4802
4803	if (!mptcp_skb_can_collapse(to, from))
4804	return false;
4805
4806	#ifdef CONFIG_TLS_DEVICE
4807	if (from->decrypted != to->decrypted)
4808	return false;
4809	#endif
4810
4811	if (!skb_try_coalesce(to, from, fragstolen, delta_truesize: &delta))
4812	return false;
4813
4814	atomic_add(i: delta, v: &sk->sk_rmem_alloc);
4815	sk_mem_charge(sk, size: delta);
4816	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOALESCE);
4817	TCP_SKB_CB(to)->end_seq = TCP_SKB_CB(from)->end_seq;
4818	TCP_SKB_CB(to)->ack_seq = TCP_SKB_CB(from)->ack_seq;
4819	TCP_SKB_CB(to)->tcp_flags |= TCP_SKB_CB(from)->tcp_flags;
4820
4821	if (TCP_SKB_CB(from)->has_rxtstamp) {
4822	TCP_SKB_CB(to)->has_rxtstamp = true;
4823	to->tstamp = from->tstamp;
4824	skb_hwtstamps(skb: to)->hwtstamp = skb_hwtstamps(skb: from)->hwtstamp;
4825	}
4826
4827	return true;
4828	}
4829
4830	static bool tcp_ooo_try_coalesce(struct sock *sk,
4831	struct sk_buff *to,
4832	struct sk_buff *from,
4833	bool *fragstolen)
4834	{
4835	bool res = tcp_try_coalesce(sk, to, from, fragstolen);
4836
4837	/* In case tcp_drop_reason() is called later, update to->gso_segs */
4838	if (res) {
4839	u32 gso_segs = max_t(u16, 1, skb_shinfo(to)->gso_segs) +
4840	max_t(u16, 1, skb_shinfo(from)->gso_segs);
4841
4842	skb_shinfo(to)->gso_segs = min_t(u32, gso_segs, 0xFFFF);
4843	}
4844	return res;
4845	}
4846
4847	static void tcp_drop_reason(struct sock *sk, struct sk_buff *skb,
4848	enum skb_drop_reason reason)
4849	{
4850	sk_drops_add(sk, skb);
4851	kfree_skb_reason(skb, reason);
4852	}
4853
4854	/* This one checks to see if we can put data from the
4855	* out_of_order queue into the receive_queue.
4856	*/
4857	static void tcp_ofo_queue(struct sock *sk)
4858	{
4859	struct tcp_sock *tp = tcp_sk(sk);
4860	__u32 dsack_high = tp->rcv_nxt;
4861	bool fin, fragstolen, eaten;
4862	struct sk_buff *skb, *tail;
4863	struct rb_node *p;
4864
4865	p = rb_first(&tp->out_of_order_queue);
4866	while (p) {
4867	skb = rb_to_skb(p);
4868	if (after(TCP_SKB_CB(skb)->seq, tp->rcv_nxt))
4869	break;
4870
4871	if (before(TCP_SKB_CB(skb)->seq, seq2: dsack_high)) {
4872	__u32 dsack = dsack_high;
4873	if (before(TCP_SKB_CB(skb)->end_seq, seq2: dsack_high))
4874	dsack_high = TCP_SKB_CB(skb)->end_seq;
4875	tcp_dsack_extend(sk, TCP_SKB_CB(skb)->seq, end_seq: dsack);
4876	}
4877	p = rb_next(p);
4878	rb_erase(&skb->rbnode, &tp->out_of_order_queue);
4879
4880	if (unlikely(!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt))) {
4881	tcp_drop_reason(sk, skb, reason: SKB_DROP_REASON_TCP_OFO_DROP);
4882	continue;
4883	}
4884
4885	tail = skb_peek_tail(list_: &sk->sk_receive_queue);
4886	eaten = tail && tcp_try_coalesce(sk, to: tail, from: skb, fragstolen: &fragstolen);
4887	tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq);
4888	fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
4889	if (!eaten)
4890	__skb_queue_tail(list: &sk->sk_receive_queue, newsk: skb);
4891	else
4892	kfree_skb_partial(skb, head_stolen: fragstolen);
4893
4894	if (unlikely(fin)) {
4895	tcp_fin(sk);
4896	/* tcp_fin() purges tp->out_of_order_queue,
4897	* so we must end this loop right now.
4898	*/
4899	break;
4900	}
4901	}
4902	}
4903
4904	static bool tcp_prune_ofo_queue(struct sock *sk, const struct sk_buff *in_skb);
4905	static int tcp_prune_queue(struct sock *sk, const struct sk_buff *in_skb);
4906
4907	static int tcp_try_rmem_schedule(struct sock *sk, struct sk_buff *skb,
4908	unsigned int size)
4909	{
4910	if (atomic_read(v: &sk->sk_rmem_alloc) > sk->sk_rcvbuf ||
4911	!sk_rmem_schedule(sk, skb, size)) {
4912
4913	if (tcp_prune_queue(sk, in_skb: skb) < 0)
4914	return -1;
4915
4916	while (!sk_rmem_schedule(sk, skb, size)) {
4917	if (!tcp_prune_ofo_queue(sk, in_skb: skb))
4918	return -1;
4919	}
4920	}
4921	return 0;
4922	}
4923
4924	static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb)
4925	{
4926	struct tcp_sock *tp = tcp_sk(sk);
4927	struct rb_node **p, *parent;
4928	struct sk_buff *skb1;
4929	u32 seq, end_seq;
4930	bool fragstolen;
4931
4932	tcp_save_lrcv_flowlabel(sk, skb);
4933	tcp_ecn_check_ce(sk, skb);
4934
4935	if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) {
4936	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFODROP);
4937	sk->sk_data_ready(sk);
4938	tcp_drop_reason(sk, skb, reason: SKB_DROP_REASON_PROTO_MEM);
4939	return;
4940	}
4941
4942	/* Disable header prediction. */
4943	tp->pred_flags = 0;
4944	inet_csk_schedule_ack(sk);
4945
4946	tp->rcv_ooopack += max_t(u16, 1, skb_shinfo(skb)->gso_segs);
4947	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOQUEUE);
4948	seq = TCP_SKB_CB(skb)->seq;
4949	end_seq = TCP_SKB_CB(skb)->end_seq;
4950
4951	p = &tp->out_of_order_queue.rb_node;
4952	if (RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
4953	/* Initial out of order segment, build 1 SACK. */
4954	if (tcp_is_sack(tp)) {
4955	tp->rx_opt.num_sacks = 1;
4956	tp->selective_acks[0].start_seq = seq;
4957	tp->selective_acks[0].end_seq = end_seq;
4958	}
4959	rb_link_node(node: &skb->rbnode, NULL, rb_link: p);
4960	rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
4961	tp->ooo_last_skb = skb;
4962	goto end;
4963	}
4964
4965	/* In the typical case, we are adding an skb to the end of the list.
4966	* Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup.
4967	*/
4968	if (tcp_ooo_try_coalesce(sk, to: tp->ooo_last_skb,
4969	from: skb, fragstolen: &fragstolen)) {
4970	coalesce_done:
4971	/* For non sack flows, do not grow window to force DUPACK
4972	* and trigger fast retransmit.
4973	*/
4974	if (tcp_is_sack(tp))
4975	tcp_grow_window(sk, skb, adjust: true);
4976	kfree_skb_partial(skb, head_stolen: fragstolen);
4977	skb = NULL;
4978	goto add_sack;
4979	}
4980	/* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */
4981	if (!before(seq1: seq, TCP_SKB_CB(tp->ooo_last_skb)->end_seq)) {
4982	parent = &tp->ooo_last_skb->rbnode;
4983	p = &parent->rb_right;
4984	goto insert;
4985	}
4986
4987	/* Find place to insert this segment. Handle overlaps on the way. */
4988	parent = NULL;
4989	while (*p) {
4990	parent = *p;
4991	skb1 = rb_to_skb(parent);
4992	if (before(seq1: seq, TCP_SKB_CB(skb1)->seq)) {
4993	p = &parent->rb_left;
4994	continue;
4995	}
4996	if (before(seq1: seq, TCP_SKB_CB(skb1)->end_seq)) {
4997	if (!after(end_seq, TCP_SKB_CB(skb1)->end_seq)) {
4998	/* All the bits are present. Drop. */
4999	NET_INC_STATS(sock_net(sk),
5000	LINUX_MIB_TCPOFOMERGE);
5001	tcp_drop_reason(sk, skb,
5002	reason: SKB_DROP_REASON_TCP_OFOMERGE);
5003	skb = NULL;
5004	tcp_dsack_set(sk, seq, end_seq);
5005	goto add_sack;
5006	}
5007	if (after(seq, TCP_SKB_CB(skb1)->seq)) {
5008	/* Partial overlap. */
5009	tcp_dsack_set(sk, seq, TCP_SKB_CB(skb1)->end_seq);
5010	} else {
5011	/* skb's seq == skb1's seq and skb covers skb1.
5012	* Replace skb1 with skb.
5013	*/
5014	rb_replace_node(victim: &skb1->rbnode, new: &skb->rbnode,
5015	root: &tp->out_of_order_queue);
5016	tcp_dsack_extend(sk,
5017	TCP_SKB_CB(skb1)->seq,
5018	TCP_SKB_CB(skb1)->end_seq);
5019	NET_INC_STATS(sock_net(sk),
5020	LINUX_MIB_TCPOFOMERGE);
5021	tcp_drop_reason(sk, skb: skb1,
5022	reason: SKB_DROP_REASON_TCP_OFOMERGE);
5023	goto merge_right;
5024	}
5025	} else if (tcp_ooo_try_coalesce(sk, to: skb1,
5026	from: skb, fragstolen: &fragstolen)) {
5027	goto coalesce_done;
5028	}
5029	p = &parent->rb_right;
5030	}
5031	insert:
5032	/* Insert segment into RB tree. */
5033	rb_link_node(node: &skb->rbnode, parent, rb_link: p);
5034	rb_insert_color(&skb->rbnode, &tp->out_of_order_queue);
5035
5036	merge_right:
5037	/* Remove other segments covered by skb. */
5038	while ((skb1 = skb_rb_next(skb)) != NULL) {
5039	if (!after(end_seq, TCP_SKB_CB(skb1)->seq))
5040	break;
5041	if (before(seq1: end_seq, TCP_SKB_CB(skb1)->end_seq)) {
5042	tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
5043	end_seq);
5044	break;
5045	}
5046	rb_erase(&skb1->rbnode, &tp->out_of_order_queue);
5047	tcp_dsack_extend(sk, TCP_SKB_CB(skb1)->seq,
5048	TCP_SKB_CB(skb1)->end_seq);
5049	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFOMERGE);
5050	tcp_drop_reason(sk, skb: skb1, reason: SKB_DROP_REASON_TCP_OFOMERGE);
5051	}
5052	/* If there is no skb after us, we are the last_skb ! */
5053	if (!skb1)
5054	tp->ooo_last_skb = skb;
5055
5056	add_sack:
5057	if (tcp_is_sack(tp))
5058	tcp_sack_new_ofo_skb(sk, seq, end_seq);
5059	end:
5060	if (skb) {
5061	/* For non sack flows, do not grow window to force DUPACK
5062	* and trigger fast retransmit.
5063	*/
5064	if (tcp_is_sack(tp))
5065	tcp_grow_window(sk, skb, adjust: false);
5066	skb_condense(skb);
5067	skb_set_owner_r(skb, sk);
5068	}
5069	}
5070
5071	static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb,
5072	bool *fragstolen)
5073	{
5074	int eaten;
5075	struct sk_buff *tail = skb_peek_tail(list_: &sk->sk_receive_queue);
5076
5077	eaten = (tail &&
5078	tcp_try_coalesce(sk, to: tail,
5079	from: skb, fragstolen)) ? 1 : 0;
5080	tcp_rcv_nxt_update(tcp_sk(sk), TCP_SKB_CB(skb)->end_seq);
5081	if (!eaten) {
5082	__skb_queue_tail(list: &sk->sk_receive_queue, newsk: skb);
5083	skb_set_owner_r(skb, sk);
5084	}
5085	return eaten;
5086	}
5087
5088	int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size)
5089	{
5090	struct sk_buff *skb;
5091	int err = -ENOMEM;
5092	int data_len = 0;
5093	bool fragstolen;
5094
5095	if (size == 0)
5096	return 0;
5097
5098	if (size > PAGE_SIZE) {
5099	int npages = min_t(size_t, size >> PAGE_SHIFT, MAX_SKB_FRAGS);
5100
5101	data_len = npages << PAGE_SHIFT;
5102	size = data_len + (size & ~PAGE_MASK);
5103	}
5104	skb = alloc_skb_with_frags(header_len: size - data_len, data_len,
5105	PAGE_ALLOC_COSTLY_ORDER,
5106	errcode: &err, gfp_mask: sk->sk_allocation);
5107	if (!skb)
5108	goto err;
5109
5110	skb_put(skb, len: size - data_len);
5111	skb->data_len = data_len;
5112	skb->len = size;
5113
5114	if (tcp_try_rmem_schedule(sk, skb, size: skb->truesize)) {
5115	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVQDROP);
5116	goto err_free;
5117	}
5118
5119	err = skb_copy_datagram_from_iter(skb, offset: 0, from: &msg->msg_iter, len: size);
5120	if (err)
5121	goto err_free;
5122
5123	TCP_SKB_CB(skb)->seq = tcp_sk(sk)->rcv_nxt;
5124	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(skb)->seq + size;
5125	TCP_SKB_CB(skb)->ack_seq = tcp_sk(sk)->snd_una - 1;
5126
5127	if (tcp_queue_rcv(sk, skb, fragstolen: &fragstolen)) {
5128	WARN_ON_ONCE(fragstolen); /* should not happen */
5129	__kfree_skb(skb);
5130	}
5131	return size;
5132
5133	err_free:
5134	kfree_skb(skb);
5135	err:
5136	return err;
5137
5138	}
5139
5140	void tcp_data_ready(struct sock *sk)
5141	{
5142	if (tcp_epollin_ready(sk, target: sk->sk_rcvlowat) || sock_flag(sk, flag: SOCK_DONE))
5143	sk->sk_data_ready(sk);
5144	}
5145
5146	static void tcp_data_queue(struct sock *sk, struct sk_buff *skb)
5147	{
5148	struct tcp_sock *tp = tcp_sk(sk);
5149	enum skb_drop_reason reason;
5150	bool fragstolen;
5151	int eaten;
5152
5153	/* If a subflow has been reset, the packet should not continue
5154	* to be processed, drop the packet.
5155	*/
5156	if (sk_is_mptcp(sk) && !mptcp_incoming_options(sk, skb)) {
5157	__kfree_skb(skb);
5158	return;
5159	}
5160
5161	if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
5162	__kfree_skb(skb);
5163	return;
5164	}
5165	skb_dst_drop(skb);
5166	__skb_pull(skb, len: tcp_hdr(skb)->doff * 4);
5167
5168	reason = SKB_DROP_REASON_NOT_SPECIFIED;
5169	tp->rx_opt.dsack = 0;
5170
5171	/* Queue data for delivery to the user.
5172	* Packets in sequence go to the receive queue.
5173	* Out of sequence packets to the out_of_order_queue.
5174	*/
5175	if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt) {
5176	if (tcp_receive_window(tp) == 0) {
5177	reason = SKB_DROP_REASON_TCP_ZEROWINDOW;
5178	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPZEROWINDOWDROP);
5179	goto out_of_window;
5180	}
5181
5182	/* Ok. In sequence. In window. */
5183	queue_and_out:
5184	if (tcp_try_rmem_schedule(sk, skb, size: skb->truesize)) {
5185	/* TODO: maybe ratelimit these WIN 0 ACK ? */
5186	inet_csk(sk)->icsk_ack.pending |=
5187	(ICSK_ACK_NOMEM | ICSK_ACK_NOW);
5188	inet_csk_schedule_ack(sk);
5189	sk->sk_data_ready(sk);
5190
5191	if (skb_queue_len(list_: &sk->sk_receive_queue)) {
5192	reason = SKB_DROP_REASON_PROTO_MEM;
5193	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVQDROP);
5194	goto drop;
5195	}
5196	sk_forced_mem_schedule(sk, size: skb->truesize);
5197	}
5198
5199	eaten = tcp_queue_rcv(sk, skb, fragstolen: &fragstolen);
5200	if (skb->len)
5201	tcp_event_data_recv(sk, skb);
5202	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
5203	tcp_fin(sk);
5204
5205	if (!RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
5206	tcp_ofo_queue(sk);
5207
5208	/* RFC5681. 4.2. SHOULD send immediate ACK, when
5209	* gap in queue is filled.
5210	*/
5211	if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
5212	inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
5213	}
5214
5215	if (tp->rx_opt.num_sacks)
5216	tcp_sack_remove(tp);
5217
5218	tcp_fast_path_check(sk);
5219
5220	if (eaten > 0)
5221	kfree_skb_partial(skb, head_stolen: fragstolen);
5222	if (!sock_flag(sk, flag: SOCK_DEAD))
5223	tcp_data_ready(sk);
5224	return;
5225	}
5226
5227	if (!after(TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt)) {
5228	tcp_rcv_spurious_retrans(sk, skb);
5229	/* A retransmit, 2nd most common case. Force an immediate ack. */
5230	reason = SKB_DROP_REASON_TCP_OLD_DATA;
5231	NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOST);
5232	tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq);
5233
5234	out_of_window:
5235	tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
5236	inet_csk_schedule_ack(sk);
5237	drop:
5238	tcp_drop_reason(sk, skb, reason);
5239	return;
5240	}
5241
5242	/* Out of window. F.e. zero window probe. */
5243	if (!before(TCP_SKB_CB(skb)->seq,
5244	seq2: tp->rcv_nxt + tcp_receive_window(tp))) {
5245	reason = SKB_DROP_REASON_TCP_OVERWINDOW;
5246	goto out_of_window;
5247	}
5248
5249	if (before(TCP_SKB_CB(skb)->seq, seq2: tp->rcv_nxt)) {
5250	/* Partial packet, seq < rcv_next < end_seq */
5251	tcp_dsack_set(sk, TCP_SKB_CB(skb)->seq, end_seq: tp->rcv_nxt);
5252
5253	/* If window is closed, drop tail of packet. But after
5254	* remembering D-SACK for its head made in previous line.
5255	*/
5256	if (!tcp_receive_window(tp)) {
5257	reason = SKB_DROP_REASON_TCP_ZEROWINDOW;
5258	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPZEROWINDOWDROP);
5259	goto out_of_window;
5260	}
5261	goto queue_and_out;
5262	}
5263
5264	tcp_data_queue_ofo(sk, skb);
5265	}
5266
5267	static struct sk_buff *tcp_skb_next(struct sk_buff *skb, struct sk_buff_head *list)
5268	{
5269	if (list)
5270	return !skb_queue_is_last(list, skb) ? skb->next : NULL;
5271
5272	return skb_rb_next(skb);
5273	}
5274
5275	static struct sk_buff *tcp_collapse_one(struct sock *sk, struct sk_buff *skb,
5276	struct sk_buff_head *list,
5277	struct rb_root *root)
5278	{
5279	struct sk_buff *next = tcp_skb_next(skb, list);
5280
5281	if (list)
5282	__skb_unlink(skb, list);
5283	else
5284	rb_erase(&skb->rbnode, root);
5285
5286	__kfree_skb(skb);
5287	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRCVCOLLAPSED);
5288
5289	return next;
5290	}
5291
5292	/* Insert skb into rb tree, ordered by TCP_SKB_CB(skb)->seq */
5293	void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb)
5294	{
5295	struct rb_node **p = &root->rb_node;
5296	struct rb_node *parent = NULL;
5297	struct sk_buff *skb1;
5298
5299	while (*p) {
5300	parent = *p;
5301	skb1 = rb_to_skb(parent);
5302	if (before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb1)->seq))
5303	p = &parent->rb_left;
5304	else
5305	p = &parent->rb_right;
5306	}
5307	rb_link_node(node: &skb->rbnode, parent, rb_link: p);
5308	rb_insert_color(&skb->rbnode, root);
5309	}
5310
5311	/* Collapse contiguous sequence of skbs head..tail with
5312	* sequence numbers start..end.
5313	*
5314	* If tail is NULL, this means until the end of the queue.
5315	*
5316	* Segments with FIN/SYN are not collapsed (only because this
5317	* simplifies code)
5318	*/
5319	static void
5320	tcp_collapse(struct sock *sk, struct sk_buff_head *list, struct rb_root *root,
5321	struct sk_buff *head, struct sk_buff *tail, u32 start, u32 end)
5322	{
5323	struct sk_buff *skb = head, *n;
5324	struct sk_buff_head tmp;
5325	bool end_of_skbs;
5326
5327	/* First, check that queue is collapsible and find
5328	* the point where collapsing can be useful.
5329	*/
5330	restart:
5331	for (end_of_skbs = true; skb != NULL && skb != tail; skb = n) {
5332	n = tcp_skb_next(skb, list);
5333
5334	/* No new bits? It is possible on ofo queue. */
5335	if (!before(seq1: start, TCP_SKB_CB(skb)->end_seq)) {
5336	skb = tcp_collapse_one(sk, skb, list, root);
5337	if (!skb)
5338	break;
5339	goto restart;
5340	}
5341
5342	/* The first skb to collapse is:
5343	* - not SYN/FIN and
5344	* - bloated or contains data before "start" or
5345	* overlaps to the next one and mptcp allow collapsing.
5346	*/
5347	if (!(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) &&
5348	(tcp_win_from_space(sk, space: skb->truesize) > skb->len ||
5349	before(TCP_SKB_CB(skb)->seq, seq2: start))) {
5350	end_of_skbs = false;
5351	break;
5352	}
5353
5354	if (n && n != tail && mptcp_skb_can_collapse(to: skb, from: n) &&
5355	TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(n)->seq) {
5356	end_of_skbs = false;
5357	break;
5358	}
5359
5360	/* Decided to skip this, advance start seq. */
5361	start = TCP_SKB_CB(skb)->end_seq;
5362	}
5363	if (end_of_skbs ||
5364	(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
5365	return;
5366
5367	__skb_queue_head_init(list: &tmp);
5368
5369	while (before(seq1: start, seq2: end)) {
5370	int copy = min_t(int, SKB_MAX_ORDER(0, 0), end - start);
5371	struct sk_buff *nskb;
5372
5373	nskb = alloc_skb(size: copy, GFP_ATOMIC);
5374	if (!nskb)
5375	break;
5376
5377	memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
5378	#ifdef CONFIG_TLS_DEVICE
5379	nskb->decrypted = skb->decrypted;
5380	#endif
5381	TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(nskb)->end_seq = start;
5382	if (list)
5383	__skb_queue_before(list, next: skb, newsk: nskb);
5384	else
5385	__skb_queue_tail(list: &tmp, newsk: nskb); /* defer rbtree insertion */
5386	skb_set_owner_r(skb: nskb, sk);
5387	mptcp_skb_ext_move(to: nskb, from: skb);
5388
5389	/* Copy data, releasing collapsed skbs. */
5390	while (copy > 0) {
5391	int offset = start - TCP_SKB_CB(skb)->seq;
5392	int size = TCP_SKB_CB(skb)->end_seq - start;
5393
5394	BUG_ON(offset < 0);
5395	if (size > 0) {
5396	size = min(copy, size);
5397	if (skb_copy_bits(skb, offset, to: skb_put(skb: nskb, len: size), len: size))
5398	BUG();
5399	TCP_SKB_CB(nskb)->end_seq += size;
5400	copy -= size;
5401	start += size;
5402	}
5403	if (!before(seq1: start, TCP_SKB_CB(skb)->end_seq)) {
5404	skb = tcp_collapse_one(sk, skb, list, root);
5405	if (!skb ||
5406	skb == tail ||
5407	!mptcp_skb_can_collapse(to: nskb, from: skb) ||
5408	(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
5409	goto end;
5410	#ifdef CONFIG_TLS_DEVICE
5411	if (skb->decrypted != nskb->decrypted)
5412	goto end;
5413	#endif
5414	}
5415	}
5416	}
5417	end:
5418	skb_queue_walk_safe(&tmp, skb, n)
5419	tcp_rbtree_insert(root, skb);
5420	}
5421
5422	/* Collapse ofo queue. Algorithm: select contiguous sequence of skbs
5423	* and tcp_collapse() them until all the queue is collapsed.
5424	*/
5425	static void tcp_collapse_ofo_queue(struct sock *sk)
5426	{
5427	struct tcp_sock *tp = tcp_sk(sk);
5428	u32 range_truesize, sum_tiny = 0;
5429	struct sk_buff *skb, *head;
5430	u32 start, end;
5431
5432	skb = skb_rb_first(&tp->out_of_order_queue);
5433	new_range:
5434	if (!skb) {
5435	tp->ooo_last_skb = skb_rb_last(&tp->out_of_order_queue);
5436	return;
5437	}
5438	start = TCP_SKB_CB(skb)->seq;
5439	end = TCP_SKB_CB(skb)->end_seq;
5440	range_truesize = skb->truesize;
5441
5442	for (head = skb;;) {
5443	skb = skb_rb_next(skb);
5444
5445	/* Range is terminated when we see a gap or when
5446	* we are at the queue end.
5447	*/
5448	if (!skb ||
5449	after(TCP_SKB_CB(skb)->seq, end) ||
5450	before(TCP_SKB_CB(skb)->end_seq, seq2: start)) {
5451	/* Do not attempt collapsing tiny skbs */
5452	if (range_truesize != head->truesize ||
5453	end - start >= SKB_WITH_OVERHEAD(PAGE_SIZE)) {
5454	tcp_collapse(sk, NULL, root: &tp->out_of_order_queue,
5455	head, tail: skb, start, end);
5456	} else {
5457	sum_tiny += range_truesize;
5458	if (sum_tiny > sk->sk_rcvbuf >> 3)
5459	return;
5460	}
5461	goto new_range;
5462	}
5463
5464	range_truesize += skb->truesize;
5465	if (unlikely(before(TCP_SKB_CB(skb)->seq, start)))
5466	start = TCP_SKB_CB(skb)->seq;
5467	if (after(TCP_SKB_CB(skb)->end_seq, end))
5468	end = TCP_SKB_CB(skb)->end_seq;
5469	}
5470	}
5471
5472	/*
5473	* Clean the out-of-order queue to make room.
5474	* We drop high sequences packets to :
5475	* 1) Let a chance for holes to be filled.
5476	* This means we do not drop packets from ooo queue if their sequence
5477	* is before incoming packet sequence.
5478	* 2) not add too big latencies if thousands of packets sit there.
5479	* (But if application shrinks SO_RCVBUF, we could still end up
5480	* freeing whole queue here)
5481	* 3) Drop at least 12.5 % of sk_rcvbuf to avoid malicious attacks.
5482	*
5483	* Return true if queue has shrunk.
5484	*/
5485	static bool tcp_prune_ofo_queue(struct sock *sk, const struct sk_buff *in_skb)
5486	{
5487	struct tcp_sock *tp = tcp_sk(sk);
5488	struct rb_node *node, *prev;
5489	bool pruned = false;
5490	int goal;
5491
5492	if (RB_EMPTY_ROOT(&tp->out_of_order_queue))
5493	return false;
5494
5495	goal = sk->sk_rcvbuf >> 3;
5496	node = &tp->ooo_last_skb->rbnode;
5497
5498	do {
5499	struct sk_buff *skb = rb_to_skb(node);
5500
5501	/* If incoming skb would land last in ofo queue, stop pruning. */
5502	if (after(TCP_SKB_CB(in_skb)->seq, TCP_SKB_CB(skb)->seq))
5503	break;
5504	pruned = true;
5505	prev = rb_prev(node);
5506	rb_erase(node, &tp->out_of_order_queue);
5507	goal -= skb->truesize;
5508	tcp_drop_reason(sk, skb, reason: SKB_DROP_REASON_TCP_OFO_QUEUE_PRUNE);
5509	tp->ooo_last_skb = rb_to_skb(prev);
5510	if (!prev || goal <= 0) {
5511	if (atomic_read(v: &sk->sk_rmem_alloc) <= sk->sk_rcvbuf &&
5512	!tcp_under_memory_pressure(sk))
5513	break;
5514	goal = sk->sk_rcvbuf >> 3;
5515	}
5516	node = prev;
5517	} while (node);
5518
5519	if (pruned) {
5520	NET_INC_STATS(sock_net(sk), LINUX_MIB_OFOPRUNED);
5521	/* Reset SACK state. A conforming SACK implementation will
5522	* do the same at a timeout based retransmit. When a connection
5523	* is in a sad state like this, we care only about integrity
5524	* of the connection not performance.
5525	*/
5526	if (tp->rx_opt.sack_ok)
5527	tcp_sack_reset(rx_opt: &tp->rx_opt);
5528	}
5529	return pruned;
5530	}
5531
5532	/* Reduce allocated memory if we can, trying to get
5533	* the socket within its memory limits again.
5534	*
5535	* Return less than zero if we should start dropping frames
5536	* until the socket owning process reads some of the data
5537	* to stabilize the situation.
5538	*/
5539	static int tcp_prune_queue(struct sock *sk, const struct sk_buff *in_skb)
5540	{
5541	struct tcp_sock *tp = tcp_sk(sk);
5542
5543	NET_INC_STATS(sock_net(sk), LINUX_MIB_PRUNECALLED);
5544
5545	if (atomic_read(v: &sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
5546	tcp_clamp_window(sk);
5547	else if (tcp_under_memory_pressure(sk))
5548	tcp_adjust_rcv_ssthresh(sk);
5549
5550	if (atomic_read(v: &sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
5551	return 0;
5552
5553	tcp_collapse_ofo_queue(sk);
5554	if (!skb_queue_empty(list: &sk->sk_receive_queue))
5555	tcp_collapse(sk, list: &sk->sk_receive_queue, NULL,
5556	head: skb_peek(list_: &sk->sk_receive_queue),
5557	NULL,
5558	start: tp->copied_seq, end: tp->rcv_nxt);
5559
5560	if (atomic_read(v: &sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
5561	return 0;
5562
5563	/* Collapsing did not help, destructive actions follow.
5564	* This must not ever occur. */
5565
5566	tcp_prune_ofo_queue(sk, in_skb);
5567
5568	if (atomic_read(v: &sk->sk_rmem_alloc) <= sk->sk_rcvbuf)
5569	return 0;
5570
5571	/* If we are really being abused, tell the caller to silently
5572	* drop receive data on the floor. It will get retransmitted
5573	* and hopefully then we'll have sufficient space.
5574	*/
5575	NET_INC_STATS(sock_net(sk), LINUX_MIB_RCVPRUNED);
5576
5577	/* Massive buffer overcommit. */
5578	tp->pred_flags = 0;
5579	return -1;
5580	}
5581
5582	static bool tcp_should_expand_sndbuf(struct sock *sk)
5583	{
5584	const struct tcp_sock *tp = tcp_sk(sk);
5585
5586	/* If the user specified a specific send buffer setting, do
5587	* not modify it.
5588	*/
5589	if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
5590	return false;
5591
5592	/* If we are under global TCP memory pressure, do not expand. */
5593	if (tcp_under_memory_pressure(sk)) {
5594	int unused_mem = sk_unused_reserved_mem(sk);
5595
5596	/* Adjust sndbuf according to reserved mem. But make sure
5597	* it never goes below SOCK_MIN_SNDBUF.
5598	* See sk_stream_moderate_sndbuf() for more details.
5599	*/
5600	if (unused_mem > SOCK_MIN_SNDBUF)
5601	WRITE_ONCE(sk->sk_sndbuf, unused_mem);
5602
5603	return false;
5604	}
5605
5606	/* If we are under soft global TCP memory pressure, do not expand. */
5607	if (sk_memory_allocated(sk) >= sk_prot_mem_limits(sk, index: 0))
5608	return false;
5609
5610	/* If we filled the congestion window, do not expand. */
5611	if (tcp_packets_in_flight(tp) >= tcp_snd_cwnd(tp))
5612	return false;
5613
5614	return true;
5615	}
5616
5617	static void tcp_new_space(struct sock *sk)
5618	{
5619	struct tcp_sock *tp = tcp_sk(sk);
5620
5621	if (tcp_should_expand_sndbuf(sk)) {
5622	tcp_sndbuf_expand(sk);
5623	tp->snd_cwnd_stamp = tcp_jiffies32;
5624	}
5625
5626	INDIRECT_CALL_1(sk->sk_write_space, sk_stream_write_space, sk);
5627	}
5628
5629	/* Caller made space either from:
5630	* 1) Freeing skbs in rtx queues (after tp->snd_una has advanced)
5631	* 2) Sent skbs from output queue (and thus advancing tp->snd_nxt)
5632	*
5633	* We might be able to generate EPOLLOUT to the application if:
5634	* 1) Space consumed in output/rtx queues is below sk->sk_sndbuf/2
5635	* 2) notsent amount (tp->write_seq - tp->snd_nxt) became
5636	* small enough that tcp_stream_memory_free() decides it
5637	* is time to generate EPOLLOUT.
5638	*/
5639	void tcp_check_space(struct sock *sk)
5640	{
5641	/* pairs with tcp_poll() */
5642	smp_mb();
5643	if (sk->sk_socket &&
5644	test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
5645	tcp_new_space(sk);
5646	if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
5647	tcp_chrono_stop(sk, type: TCP_CHRONO_SNDBUF_LIMITED);
5648	}
5649	}
5650
5651	static inline void tcp_data_snd_check(struct sock *sk)
5652	{
5653	tcp_push_pending_frames(sk);
5654	tcp_check_space(sk);
5655	}
5656
5657	/*
5658	* Check if sending an ack is needed.
5659	*/
5660	static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible)
5661	{
5662	struct tcp_sock *tp = tcp_sk(sk);
5663	unsigned long rtt, delay;
5664
5665	/* More than one full frame received... */
5666	if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss &&
5667	/* ... and right edge of window advances far enough.
5668	* (tcp_recvmsg() will send ACK otherwise).
5669	* If application uses SO_RCVLOWAT, we want send ack now if
5670	* we have not received enough bytes to satisfy the condition.
5671	*/
5672	(tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat ||
5673	__tcp_select_window(sk) >= tp->rcv_wnd)) ||
5674	/* We ACK each frame or... */
5675	tcp_in_quickack_mode(sk) ||
5676	/* Protocol state mandates a one-time immediate ACK */
5677	inet_csk(sk)->icsk_ack.pending & ICSK_ACK_NOW) {
5678	/* If we are running from __release_sock() in user context,
5679	* Defer the ack until tcp_release_cb().
5680	*/
5681	if (sock_owned_by_user_nocheck(sk) &&
5682	READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_backlog_ack_defer)) {
5683	set_bit(nr: TCP_ACK_DEFERRED, addr: &sk->sk_tsq_flags);
5684	return;
5685	}
5686	send_now:
5687	tcp_send_ack(sk);
5688	return;
5689	}
5690
5691	if (!ofo_possible || RB_EMPTY_ROOT(&tp->out_of_order_queue)) {
5692	tcp_send_delayed_ack(sk);
5693	return;
5694	}
5695
5696	if (!tcp_is_sack(tp) ||
5697	tp->compressed_ack >= READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_comp_sack_nr))
5698	goto send_now;
5699
5700	if (tp->compressed_ack_rcv_nxt != tp->rcv_nxt) {
5701	tp->compressed_ack_rcv_nxt = tp->rcv_nxt;
5702	tp->dup_ack_counter = 0;
5703	}
5704	if (tp->dup_ack_counter < TCP_FASTRETRANS_THRESH) {
5705	tp->dup_ack_counter++;
5706	goto send_now;
5707	}
5708	tp->compressed_ack++;
5709	if (hrtimer_is_queued(timer: &tp->compressed_ack_timer))
5710	return;
5711
5712	/* compress ack timer : 5 % of rtt, but no more than tcp_comp_sack_delay_ns */
5713
5714	rtt = tp->rcv_rtt_est.rtt_us;
5715	if (tp->srtt_us && tp->srtt_us < rtt)
5716	rtt = tp->srtt_us;
5717
5718	delay = min_t(unsigned long,
5719	READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_comp_sack_delay_ns),
5720	rtt * (NSEC_PER_USEC >> 3)/20);
5721	sock_hold(sk);
5722	hrtimer_start_range_ns(timer: &tp->compressed_ack_timer, tim: ns_to_ktime(ns: delay),
5723	READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_comp_sack_slack_ns),
5724	mode: HRTIMER_MODE_REL_PINNED_SOFT);
5725	}
5726
5727	static inline void tcp_ack_snd_check(struct sock *sk)
5728	{
5729	if (!inet_csk_ack_scheduled(sk)) {
5730	/* We sent a data segment already. */
5731	return;
5732	}
5733	__tcp_ack_snd_check(sk, ofo_possible: 1);
5734	}
5735
5736	/*
5737	* This routine is only called when we have urgent data
5738	* signaled. Its the 'slow' part of tcp_urg. It could be
5739	* moved inline now as tcp_urg is only called from one
5740	* place. We handle URGent data wrong. We have to - as
5741	* BSD still doesn't use the correction from RFC961.
5742	* For 1003.1g we should support a new option TCP_STDURG to permit
5743	* either form (or just set the sysctl tcp_stdurg).
5744	*/
5745
5746	static void tcp_check_urg(struct sock *sk, const struct tcphdr *th)
5747	{
5748	struct tcp_sock *tp = tcp_sk(sk);
5749	u32 ptr = ntohs(th->urg_ptr);
5750
5751	if (ptr && !READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_stdurg))
5752	ptr--;
5753	ptr += ntohl(th->seq);
5754
5755	/* Ignore urgent data that we've already seen and read. */
5756	if (after(tp->copied_seq, ptr))
5757	return;
5758
5759	/* Do not replay urg ptr.
5760	*
5761	* NOTE: interesting situation not covered by specs.
5762	* Misbehaving sender may send urg ptr, pointing to segment,
5763	* which we already have in ofo queue. We are not able to fetch
5764	* such data and will stay in TCP_URG_NOTYET until will be eaten
5765	* by recvmsg(). Seems, we are not obliged to handle such wicked
5766	* situations. But it is worth to think about possibility of some
5767	* DoSes using some hypothetical application level deadlock.
5768	*/
5769	if (before(seq1: ptr, seq2: tp->rcv_nxt))
5770	return;
5771
5772	/* Do we already have a newer (or duplicate) urgent pointer? */
5773	if (tp->urg_data && !after(ptr, tp->urg_seq))
5774	return;
5775
5776	/* Tell the world about our new urgent pointer. */
5777	sk_send_sigurg(sk);
5778
5779	/* We may be adding urgent data when the last byte read was
5780	* urgent. To do this requires some care. We cannot just ignore
5781	* tp->copied_seq since we would read the last urgent byte again
5782	* as data, nor can we alter copied_seq until this data arrives
5783	* or we break the semantics of SIOCATMARK (and thus sockatmark())
5784	*
5785	* NOTE. Double Dutch. Rendering to plain English: author of comment
5786	* above did something sort of send("A", MSG_OOB); send("B", MSG_OOB);
5787	* and expect that both A and B disappear from stream. This is _wrong_.
5788	* Though this happens in BSD with high probability, this is occasional.
5789	* Any application relying on this is buggy. Note also, that fix "works"
5790	* only in this artificial test. Insert some normal data between A and B and we will
5791	* decline of BSD again. Verdict: it is better to remove to trap
5792	* buggy users.
5793	*/
5794	if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
5795	!sock_flag(sk, flag: SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) {
5796	struct sk_buff *skb = skb_peek(list_: &sk->sk_receive_queue);
5797	tp->copied_seq++;
5798	if (skb && !before(seq1: tp->copied_seq, TCP_SKB_CB(skb)->end_seq)) {
5799	__skb_unlink(skb, list: &sk->sk_receive_queue);
5800	__kfree_skb(skb);
5801	}
5802	}
5803
5804	WRITE_ONCE(tp->urg_data, TCP_URG_NOTYET);
5805	WRITE_ONCE(tp->urg_seq, ptr);
5806
5807	/* Disable header prediction. */
5808	tp->pred_flags = 0;
5809	}
5810
5811	/* This is the 'fast' part of urgent handling. */
5812	static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *th)
5813	{
5814	struct tcp_sock *tp = tcp_sk(sk);
5815
5816	/* Check if we get a new urgent pointer - normally not. */
5817	if (unlikely(th->urg))
5818	tcp_check_urg(sk, th);
5819
5820	/* Do we wait for any urgent data? - normally not... */
5821	if (unlikely(tp->urg_data == TCP_URG_NOTYET)) {
5822	u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) -
5823	th->syn;
5824
5825	/* Is the urgent pointer pointing into this packet? */
5826	if (ptr < skb->len) {
5827	u8 tmp;
5828	if (skb_copy_bits(skb, offset: ptr, to: &tmp, len: 1))
5829	BUG();
5830	WRITE_ONCE(tp->urg_data, TCP_URG_VALID | tmp);
5831	if (!sock_flag(sk, flag: SOCK_DEAD))
5832	sk->sk_data_ready(sk);
5833	}
5834	}
5835	}
5836
5837	/* Accept RST for rcv_nxt - 1 after a FIN.
5838	* When tcp connections are abruptly terminated from Mac OSX (via ^C), a
5839	* FIN is sent followed by a RST packet. The RST is sent with the same
5840	* sequence number as the FIN, and thus according to RFC 5961 a challenge
5841	* ACK should be sent. However, Mac OSX rate limits replies to challenge
5842	* ACKs on the closed socket. In addition middleboxes can drop either the
5843	* challenge ACK or a subsequent RST.
5844	*/
5845	static bool tcp_reset_check(const struct sock *sk, const struct sk_buff *skb)
5846	{
5847	const struct tcp_sock *tp = tcp_sk(sk);
5848
5849	return unlikely(TCP_SKB_CB(skb)->seq == (tp->rcv_nxt - 1) &&
5850	(1 << sk->sk_state) & (TCPF_CLOSE_WAIT | TCPF_LAST_ACK |
5851	TCPF_CLOSING));
5852	}
5853
5854	/* Does PAWS and seqno based validation of an incoming segment, flags will
5855	* play significant role here.
5856	*/
5857	static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
5858	const struct tcphdr *th, int syn_inerr)
5859	{
5860	struct tcp_sock *tp = tcp_sk(sk);
5861	SKB_DR(reason);
5862
5863	/* RFC1323: H1. Apply PAWS check first. */
5864	if (tcp_fast_parse_options(net: sock_net(sk), skb, th, tp) &&
5865	tp->rx_opt.saw_tstamp &&
5866	tcp_paws_discard(sk, skb)) {
5867	if (!th->rst) {
5868	if (unlikely(th->syn))
5869	goto syn_challenge;
5870	NET_INC_STATS(sock_net(sk), LINUX_MIB_PAWSESTABREJECTED);
5871	if (!tcp_oow_rate_limited(net: sock_net(sk), skb,
5872	mib_idx: LINUX_MIB_TCPACKSKIPPEDPAWS,
5873	last_oow_ack_time: &tp->last_oow_ack_time))
5874	tcp_send_dupack(sk, skb);
5875	SKB_DR_SET(reason, TCP_RFC7323_PAWS);
5876	goto discard;
5877	}
5878	/* Reset is accepted even if it did not pass PAWS. */
5879	}
5880
5881	/* Step 1: check sequence number */
5882	reason = tcp_sequence(tp, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq);
5883	if (reason) {
5884	/* RFC793, page 37: "In all states except SYN-SENT, all reset
5885	* (RST) segments are validated by checking their SEQ-fields."
5886	* And page 69: "If an incoming segment is not acceptable,
5887	* an acknowledgment should be sent in reply (unless the RST
5888	* bit is set, if so drop the segment and return)".
5889	*/
5890	if (!th->rst) {
5891	if (th->syn)
5892	goto syn_challenge;
5893	if (!tcp_oow_rate_limited(net: sock_net(sk), skb,
5894	mib_idx: LINUX_MIB_TCPACKSKIPPEDSEQ,
5895	last_oow_ack_time: &tp->last_oow_ack_time))
5896	tcp_send_dupack(sk, skb);
5897	} else if (tcp_reset_check(sk, skb)) {
5898	goto reset;
5899	}
5900	goto discard;
5901	}
5902
5903	/* Step 2: check RST bit */
5904	if (th->rst) {
5905	/* RFC 5961 3.2 (extend to match against (RCV.NXT - 1) after a
5906	* FIN and SACK too if available):
5907	* If seq num matches RCV.NXT or (RCV.NXT - 1) after a FIN, or
5908	* the right-most SACK block,
5909	* then
5910	* RESET the connection
5911	* else
5912	* Send a challenge ACK
5913	*/
5914	if (TCP_SKB_CB(skb)->seq == tp->rcv_nxt ||
5915	tcp_reset_check(sk, skb))
5916	goto reset;
5917
5918	if (tcp_is_sack(tp) && tp->rx_opt.num_sacks > 0) {
5919	struct tcp_sack_block *sp = &tp->selective_acks[0];
5920	int max_sack = sp[0].end_seq;
5921	int this_sack;
5922
5923	for (this_sack = 1; this_sack < tp->rx_opt.num_sacks;
5924	++this_sack) {
5925	max_sack = after(sp[this_sack].end_seq,
5926	max_sack) ?
5927	sp[this_sack].end_seq : max_sack;
5928	}
5929
5930	if (TCP_SKB_CB(skb)->seq == max_sack)
5931	goto reset;
5932	}
5933
5934	/* Disable TFO if RST is out-of-order
5935	* and no data has been received
5936	* for current active TFO socket
5937	*/
5938	if (tp->syn_fastopen && !tp->data_segs_in &&
5939	sk->sk_state == TCP_ESTABLISHED)
5940	tcp_fastopen_active_disable(sk);
5941	tcp_send_challenge_ack(sk);
5942	SKB_DR_SET(reason, TCP_RESET);
5943	goto discard;
5944	}
5945
5946	/* step 3: check security and precedence [ignored] */
5947
5948	/* step 4: Check for a SYN
5949	* RFC 5961 4.2 : Send a challenge ack
5950	*/
5951	if (th->syn) {
5952	syn_challenge:
5953	if (syn_inerr)
5954	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
5955	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNCHALLENGE);
5956	tcp_send_challenge_ack(sk);
5957	SKB_DR_SET(reason, TCP_INVALID_SYN);
5958	goto discard;
5959	}
5960
5961	bpf_skops_parse_hdr(sk, skb);
5962
5963	return true;
5964
5965	discard:
5966	tcp_drop_reason(sk, skb, reason);
5967	return false;
5968
5969	reset:
5970	tcp_reset(sk, skb);
5971	__kfree_skb(skb);
5972	return false;
5973	}
5974
5975	/*
5976	* TCP receive function for the ESTABLISHED state.
5977	*
5978	* It is split into a fast path and a slow path. The fast path is
5979	* disabled when:
5980	* - A zero window was announced from us - zero window probing
5981	* is only handled properly in the slow path.
5982	* - Out of order segments arrived.
5983	* - Urgent data is expected.
5984	* - There is no buffer space left
5985	* - Unexpected TCP flags/window values/header lengths are received
5986	* (detected by checking the TCP header against pred_flags)
5987	* - Data is sent in both directions. Fast path only supports pure senders
5988	* or pure receivers (this means either the sequence number or the ack
5989	* value must stay constant)
5990	* - Unexpected TCP option.
5991	*
5992	* When these conditions are not satisfied it drops into a standard
5993	* receive procedure patterned after RFC793 to handle all cases.
5994	* The first three cases are guaranteed by proper pred_flags setting,
5995	* the rest is checked inline. Fast processing is turned on in
5996	* tcp_data_queue when everything is OK.
5997	*/
5998	void tcp_rcv_established(struct sock *sk, struct sk_buff *skb)
5999	{
6000	enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
6001	const struct tcphdr *th = (const struct tcphdr *)skb->data;
6002	struct tcp_sock *tp = tcp_sk(sk);
6003	unsigned int len = skb->len;
6004
6005	/* TCP congestion window tracking */
6006	trace_tcp_probe(sk, skb);
6007
6008	tcp_mstamp_refresh(tp);
6009	if (unlikely(!rcu_access_pointer(sk->sk_rx_dst)))
6010	inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb);
6011	/*
6012	* Header prediction.
6013	* The code loosely follows the one in the famous
6014	* "30 instruction TCP receive" Van Jacobson mail.
6015	*
6016	* Van's trick is to deposit buffers into socket queue
6017	* on a device interrupt, to call tcp_recv function
6018	* on the receive process context and checksum and copy
6019	* the buffer to user space. smart...
6020	*
6021	* Our current scheme is not silly either but we take the
6022	* extra cost of the net_bh soft interrupt processing...
6023	* We do checksum and copy also but from device to kernel.
6024	*/
6025
6026	tp->rx_opt.saw_tstamp = 0;
6027
6028	/* pred_flags is 0xS?10 << 16 + snd_wnd
6029	* if header_prediction is to be made
6030	* 'S' will always be tp->tcp_header_len >> 2
6031	* '?' will be 0 for the fast path, otherwise pred_flags is 0 to
6032	* turn it off (when there are holes in the receive
6033	* space for instance)
6034	* PSH flag is ignored.
6035	*/
6036
6037	if ((tcp_flag_word(th) & TCP_HP_BITS) == tp->pred_flags &&
6038	TCP_SKB_CB(skb)->seq == tp->rcv_nxt &&
6039	!after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) {
6040	int tcp_header_len = tp->tcp_header_len;
6041
6042	/* Timestamp header prediction: tcp_header_len
6043	* is automatically equal to th->doff*4 due to pred_flags
6044	* match.
6045	*/
6046
6047	/* Check timestamp */
6048	if (tcp_header_len == sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) {
6049	/* No? Slow path! */
6050	if (!tcp_parse_aligned_timestamp(tp, th))
6051	goto slow_path;
6052
6053	/* If PAWS failed, check it more carefully in slow path */
6054	if ((s32)(tp->rx_opt.rcv_tsval - tp->rx_opt.ts_recent) < 0)
6055	goto slow_path;
6056
6057	/* DO NOT update ts_recent here, if checksum fails
6058	* and timestamp was corrupted part, it will result
6059	* in a hung connection since we will drop all
6060	* future packets due to the PAWS test.
6061	*/
6062	}
6063
6064	if (len <= tcp_header_len) {
6065	/* Bulk data transfer: sender */
6066	if (len == tcp_header_len) {
6067	/* Predicted packet is in window by definition.
6068	* seq == rcv_nxt and rcv_wup <= rcv_nxt.
6069	* Hence, check seq<=rcv_wup reduces to:
6070	*/
6071	if (tcp_header_len ==
6072	(sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
6073	tp->rcv_nxt == tp->rcv_wup)
6074	tcp_store_ts_recent(tp);
6075
6076	/* We know that such packets are checksummed
6077	* on entry.
6078	*/
6079	tcp_ack(sk, skb, flag: 0);
6080	__kfree_skb(skb);
6081	tcp_data_snd_check(sk);
6082	/* When receiving pure ack in fast path, update
6083	* last ts ecr directly instead of calling
6084	* tcp_rcv_rtt_measure_ts()
6085	*/
6086	tp->rcv_rtt_last_tsecr = tp->rx_opt.rcv_tsecr;
6087	return;
6088	} else { /* Header too small */
6089	reason = SKB_DROP_REASON_PKT_TOO_SMALL;
6090	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
6091	goto discard;
6092	}
6093	} else {
6094	int eaten = 0;
6095	bool fragstolen = false;
6096
6097	if (tcp_checksum_complete(skb))
6098	goto csum_error;
6099
6100	if ((int)skb->truesize > sk->sk_forward_alloc)
6101	goto step5;
6102
6103	/* Predicted packet is in window by definition.
6104	* seq == rcv_nxt and rcv_wup <= rcv_nxt.
6105	* Hence, check seq<=rcv_wup reduces to:
6106	*/
6107	if (tcp_header_len ==
6108	(sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED) &&
6109	tp->rcv_nxt == tp->rcv_wup)
6110	tcp_store_ts_recent(tp);
6111
6112	tcp_rcv_rtt_measure_ts(sk, skb);
6113
6114	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPHPHITS);
6115
6116	/* Bulk data transfer: receiver */
6117	skb_dst_drop(skb);
6118	__skb_pull(skb, len: tcp_header_len);
6119	eaten = tcp_queue_rcv(sk, skb, fragstolen: &fragstolen);
6120
6121	tcp_event_data_recv(sk, skb);
6122
6123	if (TCP_SKB_CB(skb)->ack_seq != tp->snd_una) {
6124	/* Well, only one small jumplet in fast path... */
6125	tcp_ack(sk, skb, FLAG_DATA);
6126	tcp_data_snd_check(sk);
6127	if (!inet_csk_ack_scheduled(sk))
6128	goto no_ack;
6129	} else {
6130	tcp_update_wl(tp, TCP_SKB_CB(skb)->seq);
6131	}
6132
6133	__tcp_ack_snd_check(sk, ofo_possible: 0);
6134	no_ack:
6135	if (eaten)
6136	kfree_skb_partial(skb, head_stolen: fragstolen);
6137	tcp_data_ready(sk);
6138	return;
6139	}
6140	}
6141
6142	slow_path:
6143	if (len < (th->doff << 2) || tcp_checksum_complete(skb))
6144	goto csum_error;
6145
6146	if (!th->ack && !th->rst && !th->syn) {
6147	reason = SKB_DROP_REASON_TCP_FLAGS;
6148	goto discard;
6149	}
6150
6151	/*
6152	* Standard slow path.
6153	*/
6154
6155	if (!tcp_validate_incoming(sk, skb, th, syn_inerr: 1))
6156	return;
6157
6158	step5:
6159	reason = tcp_ack(sk, skb, FLAG_SLOWPATH | FLAG_UPDATE_TS_RECENT);
6160	if ((int)reason < 0) {
6161	reason = -reason;
6162	goto discard;
6163	}
6164	tcp_rcv_rtt_measure_ts(sk, skb);
6165
6166	/* Process urgent data. */
6167	tcp_urg(sk, skb, th);
6168
6169	/* step 7: process the segment text */
6170	tcp_data_queue(sk, skb);
6171
6172	tcp_data_snd_check(sk);
6173	tcp_ack_snd_check(sk);
6174	return;
6175
6176	csum_error:
6177	reason = SKB_DROP_REASON_TCP_CSUM;
6178	trace_tcp_bad_csum(skb);
6179	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
6180	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
6181
6182	discard:
6183	tcp_drop_reason(sk, skb, reason);
6184	}
6185	EXPORT_SYMBOL(tcp_rcv_established);
6186
6187	void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb)
6188	{
6189	struct inet_connection_sock *icsk = inet_csk(sk);
6190	struct tcp_sock *tp = tcp_sk(sk);
6191
6192	tcp_mtup_init(sk);
6193	icsk->icsk_af_ops->rebuild_header(sk);
6194	tcp_init_metrics(sk);
6195
6196	/* Initialize the congestion window to start the transfer.
6197	* Cut cwnd down to 1 per RFC5681 if SYN or SYN-ACK has been
6198	* retransmitted. In light of RFC6298 more aggressive 1sec
6199	* initRTO, we only reset cwnd when more than 1 SYN/SYN-ACK
6200	* retransmission has occurred.
6201	*/
6202	if (tp->total_retrans > 1 && tp->undo_marker)
6203	tcp_snd_cwnd_set(tp, val: 1);
6204	else
6205	tcp_snd_cwnd_set(tp, val: tcp_init_cwnd(tp, dst: __sk_dst_get(sk)));
6206	tp->snd_cwnd_stamp = tcp_jiffies32;
6207
6208	bpf_skops_established(sk, bpf_op, skb);
6209	/* Initialize congestion control unless BPF initialized it already: */
6210	if (!icsk->icsk_ca_initialized)
6211	tcp_init_congestion_control(sk);
6212	tcp_init_buffer_space(sk);
6213	}
6214
6215	void tcp_finish_connect(struct sock *sk, struct sk_buff *skb)
6216	{
6217	struct tcp_sock *tp = tcp_sk(sk);
6218	struct inet_connection_sock *icsk = inet_csk(sk);
6219
6220	tcp_ao_finish_connect(sk, skb);
6221	tcp_set_state(sk, state: TCP_ESTABLISHED);
6222	icsk->icsk_ack.lrcvtime = tcp_jiffies32;
6223
6224	if (skb) {
6225	icsk->icsk_af_ops->sk_rx_dst_set(sk, skb);
6226	security_inet_conn_established(sk, skb);
6227	sk_mark_napi_id(sk, skb);
6228	}
6229
6230	tcp_init_transfer(sk, bpf_op: BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB, skb);
6231
6232	/* Prevent spurious tcp_cwnd_restart() on first data
6233	* packet.
6234	*/
6235	tp->lsndtime = tcp_jiffies32;
6236
6237	if (sock_flag(sk, flag: SOCK_KEEPOPEN))
6238	inet_csk_reset_keepalive_timer(sk, timeout: keepalive_time_when(tp));
6239
6240	if (!tp->rx_opt.snd_wscale)
6241	__tcp_fast_path_on(tp, snd_wnd: tp->snd_wnd);
6242	else
6243	tp->pred_flags = 0;
6244	}
6245
6246	static bool tcp_rcv_fastopen_synack(struct sock *sk, struct sk_buff *synack,
6247	struct tcp_fastopen_cookie *cookie)
6248	{
6249	struct tcp_sock *tp = tcp_sk(sk);
6250	struct sk_buff *data = tp->syn_data ? tcp_rtx_queue_head(sk) : NULL;
6251	u16 mss = tp->rx_opt.mss_clamp, try_exp = 0;
6252	bool syn_drop = false;
6253
6254	if (mss == tp->rx_opt.user_mss) {
6255	struct tcp_options_received opt;
6256
6257	/* Get original SYNACK MSS value if user MSS sets mss_clamp */
6258	tcp_clear_options(rx_opt: &opt);
6259	opt.user_mss = opt.mss_clamp = 0;
6260	tcp_parse_options(sock_net(sk), synack, &opt, 0, NULL);
6261	mss = opt.mss_clamp;
6262	}
6263
6264	if (!tp->syn_fastopen) {
6265	/* Ignore an unsolicited cookie */
6266	cookie->len = -1;
6267	} else if (tp->total_retrans) {
6268	/* SYN timed out and the SYN-ACK neither has a cookie nor
6269	* acknowledges data. Presumably the remote received only
6270	* the retransmitted (regular) SYNs: either the original
6271	* SYN-data or the corresponding SYN-ACK was dropped.
6272	*/
6273	syn_drop = (cookie->len < 0 && data);
6274	} else if (cookie->len < 0 && !tp->syn_data) {
6275	/* We requested a cookie but didn't get it. If we did not use
6276	* the (old) exp opt format then try so next time (try_exp=1).
6277	* Otherwise we go back to use the RFC7413 opt (try_exp=2).
6278	*/
6279	try_exp = tp->syn_fastopen_exp ? 2 : 1;
6280	}
6281
6282	tcp_fastopen_cache_set(sk, mss, cookie, syn_lost: syn_drop, try_exp);
6283
6284	if (data) { /* Retransmit unacked data in SYN */
6285	if (tp->total_retrans)
6286	tp->fastopen_client_fail = TFO_SYN_RETRANSMITTED;
6287	else
6288	tp->fastopen_client_fail = TFO_DATA_NOT_ACKED;
6289	skb_rbtree_walk_from(data)
6290	tcp_mark_skb_lost(sk, skb: data);
6291	tcp_xmit_retransmit_queue(sk);
6292	NET_INC_STATS(sock_net(sk),
6293	LINUX_MIB_TCPFASTOPENACTIVEFAIL);
6294	return true;
6295	}
6296	tp->syn_data_acked = tp->syn_data;
6297	if (tp->syn_data_acked) {
6298	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE);
6299	/* SYN-data is counted as two separate packets in tcp_ack() */
6300	if (tp->delivered > 1)
6301	--tp->delivered;
6302	}
6303
6304	tcp_fastopen_add_skb(sk, skb: synack);
6305
6306	return false;
6307	}
6308
6309	static void smc_check_reset_syn(struct tcp_sock *tp)
6310	{
6311	#if IS_ENABLED(CONFIG_SMC)
6312	if (static_branch_unlikely(&tcp_have_smc)) {
6313	if (tp->syn_smc && !tp->rx_opt.smc_ok)
6314	tp->syn_smc = 0;
6315	}
6316	#endif
6317	}
6318
6319	static void tcp_try_undo_spurious_syn(struct sock *sk)
6320	{
6321	struct tcp_sock *tp = tcp_sk(sk);
6322	u32 syn_stamp;
6323
6324	/* undo_marker is set when SYN or SYNACK times out. The timeout is
6325	* spurious if the ACK's timestamp option echo value matches the
6326	* original SYN timestamp.
6327	*/
6328	syn_stamp = tp->retrans_stamp;
6329	if (tp->undo_marker && syn_stamp && tp->rx_opt.saw_tstamp &&
6330	syn_stamp == tp->rx_opt.rcv_tsecr)
6331	tp->undo_marker = 0;
6332	}
6333
6334	static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb,
6335	const struct tcphdr *th)
6336	{
6337	struct inet_connection_sock *icsk = inet_csk(sk);
6338	struct tcp_sock *tp = tcp_sk(sk);
6339	struct tcp_fastopen_cookie foc = { .len = -1 };
6340	int saved_clamp = tp->rx_opt.mss_clamp;
6341	bool fastopen_fail;
6342	SKB_DR(reason);
6343
6344	tcp_parse_options(sock_net(sk), skb, &tp->rx_opt, 0, &foc);
6345	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr)
6346	tp->rx_opt.rcv_tsecr -= tp->tsoffset;
6347
6348	if (th->ack) {
6349	/* rfc793:
6350	* "If the state is SYN-SENT then
6351	* first check the ACK bit
6352	* If the ACK bit is set
6353	* If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send
6354	* a reset (unless the RST bit is set, if so drop
6355	* the segment and return)"
6356	*/
6357	if (!after(TCP_SKB_CB(skb)->ack_seq, tp->snd_una) ||
6358	after(TCP_SKB_CB(skb)->ack_seq, tp->snd_nxt)) {
6359	/* Previous FIN/ACK or RST/ACK might be ignored. */
6360	if (icsk->icsk_retransmits == 0)
6361	inet_csk_reset_xmit_timer(sk,
6362	ICSK_TIME_RETRANS,
6363	TCP_TIMEOUT_MIN, TCP_RTO_MAX);
6364	SKB_DR_SET(reason, TCP_INVALID_ACK_SEQUENCE);
6365	goto reset_and_undo;
6366	}
6367
6368	if (tp->rx_opt.saw_tstamp && tp->rx_opt.rcv_tsecr &&
6369	!between(seq1: tp->rx_opt.rcv_tsecr, seq2: tp->retrans_stamp,
6370	seq3: tcp_time_stamp_ts(tp))) {
6371	NET_INC_STATS(sock_net(sk),
6372	LINUX_MIB_PAWSACTIVEREJECTED);
6373	SKB_DR_SET(reason, TCP_RFC7323_PAWS);
6374	goto reset_and_undo;
6375	}
6376
6377	/* Now ACK is acceptable.
6378	*
6379	* "If the RST bit is set
6380	* If the ACK was acceptable then signal the user "error:
6381	* connection reset", drop the segment, enter CLOSED state,
6382	* delete TCB, and return."
6383	*/
6384
6385	if (th->rst) {
6386	tcp_reset(sk, skb);
6387	consume:
6388	__kfree_skb(skb);
6389	return 0;
6390	}
6391
6392	/* rfc793:
6393	* "fifth, if neither of the SYN or RST bits is set then
6394	* drop the segment and return."
6395	*
6396	* See note below!
6397	* --ANK(990513)
6398	*/
6399	if (!th->syn) {
6400	SKB_DR_SET(reason, TCP_FLAGS);
6401	goto discard_and_undo;
6402	}
6403	/* rfc793:
6404	* "If the SYN bit is on ...
6405	* are acceptable then ...
6406	* (our SYN has been ACKed), change the connection
6407	* state to ESTABLISHED..."
6408	*/
6409
6410	tcp_ecn_rcv_synack(tp, th);
6411
6412	tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
6413	tcp_try_undo_spurious_syn(sk);
6414	tcp_ack(sk, skb, FLAG_SLOWPATH);
6415
6416	/* Ok.. it's good. Set up sequence numbers and
6417	* move to established.
6418	*/
6419	WRITE_ONCE(tp->rcv_nxt, TCP_SKB_CB(skb)->seq + 1);
6420	tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;
6421
6422	/* RFC1323: The window in SYN & SYN/ACK segments is
6423	* never scaled.
6424	*/
6425	tp->snd_wnd = ntohs(th->window);
6426
6427	if (!tp->rx_opt.wscale_ok) {
6428	tp->rx_opt.snd_wscale = tp->rx_opt.rcv_wscale = 0;
6429	tp->window_clamp = min(tp->window_clamp, 65535U);
6430	}
6431
6432	if (tp->rx_opt.saw_tstamp) {
6433	tp->rx_opt.tstamp_ok = 1;
6434	tp->tcp_header_len =
6435	sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
6436	tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;
6437	tcp_store_ts_recent(tp);
6438	} else {
6439	tp->tcp_header_len = sizeof(struct tcphdr);
6440	}
6441
6442	tcp_sync_mss(sk, pmtu: icsk->icsk_pmtu_cookie);
6443	tcp_initialize_rcv_mss(sk);
6444
6445	/* Remember, tcp_poll() does not lock socket!
6446	* Change state from SYN-SENT only after copied_seq
6447	* is initialized. */
6448	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
6449
6450	smc_check_reset_syn(tp);
6451
6452	smp_mb();
6453
6454	tcp_finish_connect(sk, skb);
6455
6456	fastopen_fail = (tp->syn_fastopen || tp->syn_data) &&
6457	tcp_rcv_fastopen_synack(sk, synack: skb, cookie: &foc);
6458
6459	if (!sock_flag(sk, flag: SOCK_DEAD)) {
6460	sk->sk_state_change(sk);
6461	sk_wake_async(sk, how: SOCK_WAKE_IO, POLL_OUT);
6462	}
6463	if (fastopen_fail)
6464	return -1;
6465	if (sk->sk_write_pending ||
6466	READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept) ||
6467	inet_csk_in_pingpong_mode(sk)) {
6468	/* Save one ACK. Data will be ready after
6469	* several ticks, if write_pending is set.
6470	*
6471	* It may be deleted, but with this feature tcpdumps
6472	* look so _wonderfully_ clever, that I was not able
6473	* to stand against the temptation 8) --ANK
6474	*/
6475	inet_csk_schedule_ack(sk);
6476	tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
6477	inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
6478	TCP_DELACK_MAX, TCP_RTO_MAX);
6479	goto consume;
6480	}
6481	tcp_send_ack(sk);
6482	return -1;
6483	}
6484
6485	/* No ACK in the segment */
6486
6487	if (th->rst) {
6488	/* rfc793:
6489	* "If the RST bit is set
6490	*
6491	* Otherwise (no ACK) drop the segment and return."
6492	*/
6493	SKB_DR_SET(reason, TCP_RESET);
6494	goto discard_and_undo;
6495	}
6496
6497	/* PAWS check. */
6498	if (tp->rx_opt.ts_recent_stamp && tp->rx_opt.saw_tstamp &&
6499	tcp_paws_reject(rx_opt: &tp->rx_opt, rst: 0)) {
6500	SKB_DR_SET(reason, TCP_RFC7323_PAWS);
6501	goto discard_and_undo;
6502	}
6503	if (th->syn) {
6504	/* We see SYN without ACK. It is attempt of
6505	* simultaneous connect with crossed SYNs.
6506	* Particularly, it can be connect to self.
6507	*/
6508	#ifdef CONFIG_TCP_AO
6509	struct tcp_ao_info *ao;
6510
6511	ao = rcu_dereference_protected(tp->ao_info,
6512	lockdep_sock_is_held(sk));
6513	if (ao) {
6514	WRITE_ONCE(ao->risn, th->seq);
6515	ao->rcv_sne = 0;
6516	}
6517	#endif
6518	tcp_set_state(sk, state: TCP_SYN_RECV);
6519
6520	if (tp->rx_opt.saw_tstamp) {
6521	tp->rx_opt.tstamp_ok = 1;
6522	tcp_store_ts_recent(tp);
6523	tp->tcp_header_len =
6524	sizeof(struct tcphdr) + TCPOLEN_TSTAMP_ALIGNED;
6525	} else {
6526	tp->tcp_header_len = sizeof(struct tcphdr);
6527	}
6528
6529	WRITE_ONCE(tp->rcv_nxt, TCP_SKB_CB(skb)->seq + 1);
6530	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
6531	tp->rcv_wup = TCP_SKB_CB(skb)->seq + 1;
6532
6533	/* RFC1323: The window in SYN & SYN/ACK segments is
6534	* never scaled.
6535	*/
6536	tp->snd_wnd = ntohs(th->window);
6537	tp->snd_wl1 = TCP_SKB_CB(skb)->seq;
6538	tp->max_window = tp->snd_wnd;
6539
6540	tcp_ecn_rcv_syn(tp, th);
6541
6542	tcp_mtup_init(sk);
6543	tcp_sync_mss(sk, pmtu: icsk->icsk_pmtu_cookie);
6544	tcp_initialize_rcv_mss(sk);
6545
6546	tcp_send_synack(sk);
6547	#if 0
6548	/* Note, we could accept data and URG from this segment.
6549	* There are no obstacles to make this (except that we must
6550	* either change tcp_recvmsg() to prevent it from returning data
6551	* before 3WHS completes per RFC793, or employ TCP Fast Open).
6552	*
6553	* However, if we ignore data in ACKless segments sometimes,
6554	* we have no reasons to accept it sometimes.
6555	* Also, seems the code doing it in step6 of tcp_rcv_state_process
6556	* is not flawless. So, discard packet for sanity.
6557	* Uncomment this return to process the data.
6558	*/
6559	return -1;
6560	#else
6561	goto consume;
6562	#endif
6563	}
6564	/* "fifth, if neither of the SYN or RST bits is set then
6565	* drop the segment and return."
6566	*/
6567
6568	discard_and_undo:
6569	tcp_clear_options(rx_opt: &tp->rx_opt);
6570	tp->rx_opt.mss_clamp = saved_clamp;
6571	tcp_drop_reason(sk, skb, reason);
6572	return 0;
6573
6574	reset_and_undo:
6575	tcp_clear_options(rx_opt: &tp->rx_opt);
6576	tp->rx_opt.mss_clamp = saved_clamp;
6577	/* we can reuse/return @reason to its caller to handle the exception */
6578	return reason;
6579	}
6580
6581	static void tcp_rcv_synrecv_state_fastopen(struct sock *sk)
6582	{
6583	struct tcp_sock *tp = tcp_sk(sk);
6584	struct request_sock *req;
6585
6586	/* If we are still handling the SYNACK RTO, see if timestamp ECR allows
6587	* undo. If peer SACKs triggered fast recovery, we can't undo here.
6588	*/
6589	if (inet_csk(sk)->icsk_ca_state == TCP_CA_Loss && !tp->packets_out)
6590	tcp_try_undo_recovery(sk);
6591
6592	/* Reset rtx states to prevent spurious retransmits_timed_out() */
6593	tcp_update_rto_time(tp);
6594	tp->retrans_stamp = 0;
6595	inet_csk(sk)->icsk_retransmits = 0;
6596
6597	/* Once we leave TCP_SYN_RECV or TCP_FIN_WAIT_1,
6598	* we no longer need req so release it.
6599	*/
6600	req = rcu_dereference_protected(tp->fastopen_rsk,
6601	lockdep_sock_is_held(sk));
6602	reqsk_fastopen_remove(sk, req, reset: false);
6603
6604	/* Re-arm the timer because data may have been sent out.
6605	* This is similar to the regular data transmission case
6606	* when new data has just been ack'ed.
6607	*
6608	* (TFO) - we could try to be more aggressive and
6609	* retransmitting any data sooner based on when they
6610	* are sent out.
6611	*/
6612	tcp_rearm_rto(sk);
6613	}
6614
6615	/*
6616	* This function implements the receiving procedure of RFC 793 for
6617	* all states except ESTABLISHED and TIME_WAIT.
6618	* It's called from both tcp_v4_rcv and tcp_v6_rcv and should be
6619	* address independent.
6620	*/
6621
6622	enum skb_drop_reason
6623	tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb)
6624	{
6625	struct tcp_sock *tp = tcp_sk(sk);
6626	struct inet_connection_sock *icsk = inet_csk(sk);
6627	const struct tcphdr *th = tcp_hdr(skb);
6628	struct request_sock *req;
6629	int queued = 0;
6630	SKB_DR(reason);
6631
6632	switch (sk->sk_state) {
6633	case TCP_CLOSE:
6634	SKB_DR_SET(reason, TCP_CLOSE);
6635	goto discard;
6636
6637	case TCP_LISTEN:
6638	if (th->ack)
6639	return SKB_DROP_REASON_TCP_FLAGS;
6640
6641	if (th->rst) {
6642	SKB_DR_SET(reason, TCP_RESET);
6643	goto discard;
6644	}
6645	if (th->syn) {
6646	if (th->fin) {
6647	SKB_DR_SET(reason, TCP_FLAGS);
6648	goto discard;
6649	}
6650	/* It is possible that we process SYN packets from backlog,
6651	* so we need to make sure to disable BH and RCU right there.
6652	*/
6653	rcu_read_lock();
6654	local_bh_disable();
6655	icsk->icsk_af_ops->conn_request(sk, skb);
6656	local_bh_enable();
6657	rcu_read_unlock();
6658
6659	consume_skb(skb);
6660	return 0;
6661	}
6662	SKB_DR_SET(reason, TCP_FLAGS);
6663	goto discard;
6664
6665	case TCP_SYN_SENT:
6666	tp->rx_opt.saw_tstamp = 0;
6667	tcp_mstamp_refresh(tp);
6668	queued = tcp_rcv_synsent_state_process(sk, skb, th);
6669	if (queued >= 0)
6670	return queued;
6671
6672	/* Do step6 onward by hand. */
6673	tcp_urg(sk, skb, th);
6674	__kfree_skb(skb);
6675	tcp_data_snd_check(sk);
6676	return 0;
6677	}
6678
6679	tcp_mstamp_refresh(tp);
6680	tp->rx_opt.saw_tstamp = 0;
6681	req = rcu_dereference_protected(tp->fastopen_rsk,
6682	lockdep_sock_is_held(sk));
6683	if (req) {
6684	bool req_stolen;
6685
6686	WARN_ON_ONCE(sk->sk_state != TCP_SYN_RECV &&
6687	sk->sk_state != TCP_FIN_WAIT1);
6688
6689	if (!tcp_check_req(sk, skb, req, fastopen: true, lost_race: &req_stolen)) {
6690	SKB_DR_SET(reason, TCP_FASTOPEN);
6691	goto discard;
6692	}
6693	}
6694
6695	if (!th->ack && !th->rst && !th->syn) {
6696	SKB_DR_SET(reason, TCP_FLAGS);
6697	goto discard;
6698	}
6699	if (!tcp_validate_incoming(sk, skb, th, syn_inerr: 0))
6700	return 0;
6701
6702	/* step 5: check the ACK field */
6703	reason = tcp_ack(sk, skb, FLAG_SLOWPATH |
6704	FLAG_UPDATE_TS_RECENT |
6705	FLAG_NO_CHALLENGE_ACK);
6706
6707	if ((int)reason <= 0) {
6708	if (sk->sk_state == TCP_SYN_RECV) {
6709	/* send one RST */
6710	if (!reason)
6711	return SKB_DROP_REASON_TCP_OLD_ACK;
6712	return -reason;
6713	}
6714	/* accept old ack during closing */
6715	if ((int)reason < 0) {
6716	tcp_send_challenge_ack(sk);
6717	reason = -reason;
6718	goto discard;
6719	}
6720	}
6721	SKB_DR_SET(reason, NOT_SPECIFIED);
6722	switch (sk->sk_state) {
6723	case TCP_SYN_RECV:
6724	tp->delivered++; /* SYN-ACK delivery isn't tracked in tcp_ack */
6725	if (!tp->srtt_us)
6726	tcp_synack_rtt_meas(sk, req);
6727
6728	if (req) {
6729	tcp_rcv_synrecv_state_fastopen(sk);
6730	} else {
6731	tcp_try_undo_spurious_syn(sk);
6732	tp->retrans_stamp = 0;
6733	tcp_init_transfer(sk, bpf_op: BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB,
6734	skb);
6735	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
6736	}
6737	tcp_ao_established(sk);
6738	smp_mb();
6739	tcp_set_state(sk, state: TCP_ESTABLISHED);
6740	sk->sk_state_change(sk);
6741
6742	/* Note, that this wakeup is only for marginal crossed SYN case.
6743	* Passively open sockets are not waked up, because
6744	* sk->sk_sleep == NULL and sk->sk_socket == NULL.
6745	*/
6746	if (sk->sk_socket)
6747	sk_wake_async(sk, how: SOCK_WAKE_IO, POLL_OUT);
6748
6749	tp->snd_una = TCP_SKB_CB(skb)->ack_seq;
6750	tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale;
6751	tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
6752
6753	if (tp->rx_opt.tstamp_ok)
6754	tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;
6755
6756	if (!inet_csk(sk)->icsk_ca_ops->cong_control)
6757	tcp_update_pacing_rate(sk);
6758
6759	/* Prevent spurious tcp_cwnd_restart() on first data packet */
6760	tp->lsndtime = tcp_jiffies32;
6761
6762	tcp_initialize_rcv_mss(sk);
6763	tcp_fast_path_on(tp);
6764	break;
6765
6766	case TCP_FIN_WAIT1: {
6767	int tmo;
6768
6769	if (req)
6770	tcp_rcv_synrecv_state_fastopen(sk);
6771
6772	if (tp->snd_una != tp->write_seq)
6773	break;
6774
6775	tcp_set_state(sk, state: TCP_FIN_WAIT2);
6776	WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | SEND_SHUTDOWN);
6777
6778	sk_dst_confirm(sk);
6779
6780	if (!sock_flag(sk, flag: SOCK_DEAD)) {
6781	/* Wake up lingering close() */
6782	sk->sk_state_change(sk);
6783	break;
6784	}
6785
6786	if (READ_ONCE(tp->linger2) < 0) {
6787	tcp_done(sk);
6788	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
6789	return SKB_DROP_REASON_TCP_ABORT_ON_DATA;
6790	}
6791	if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
6792	after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
6793	/* Receive out of order FIN after close() */
6794	if (tp->syn_fastopen && th->fin)
6795	tcp_fastopen_active_disable(sk);
6796	tcp_done(sk);
6797	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
6798	return SKB_DROP_REASON_TCP_ABORT_ON_DATA;
6799	}
6800
6801	tmo = tcp_fin_time(sk);
6802	if (tmo > TCP_TIMEWAIT_LEN) {
6803	inet_csk_reset_keepalive_timer(sk, timeout: tmo - TCP_TIMEWAIT_LEN);
6804	} else if (th->fin || sock_owned_by_user(sk)) {
6805	/* Bad case. We could lose such FIN otherwise.
6806	* It is not a big problem, but it looks confusing
6807	* and not so rare event. We still can lose it now,
6808	* if it spins in bh_lock_sock(), but it is really
6809	* marginal case.
6810	*/
6811	inet_csk_reset_keepalive_timer(sk, timeout: tmo);
6812	} else {
6813	tcp_time_wait(sk, state: TCP_FIN_WAIT2, timeo: tmo);
6814	goto consume;
6815	}
6816	break;
6817	}
6818
6819	case TCP_CLOSING:
6820	if (tp->snd_una == tp->write_seq) {
6821	tcp_time_wait(sk, state: TCP_TIME_WAIT, timeo: 0);
6822	goto consume;
6823	}
6824	break;
6825
6826	case TCP_LAST_ACK:
6827	if (tp->snd_una == tp->write_seq) {
6828	tcp_update_metrics(sk);
6829	tcp_done(sk);
6830	goto consume;
6831	}
6832	break;
6833	}
6834
6835	/* step 6: check the URG bit */
6836	tcp_urg(sk, skb, th);
6837
6838	/* step 7: process the segment text */
6839	switch (sk->sk_state) {
6840	case TCP_CLOSE_WAIT:
6841	case TCP_CLOSING:
6842	case TCP_LAST_ACK:
6843	if (!before(TCP_SKB_CB(skb)->seq, seq2: tp->rcv_nxt)) {
6844	/* If a subflow has been reset, the packet should not
6845	* continue to be processed, drop the packet.
6846	*/
6847	if (sk_is_mptcp(sk) && !mptcp_incoming_options(sk, skb))
6848	goto discard;
6849	break;
6850	}
6851	fallthrough;
6852	case TCP_FIN_WAIT1:
6853	case TCP_FIN_WAIT2:
6854	/* RFC 793 says to queue data in these states,
6855	* RFC 1122 says we MUST send a reset.
6856	* BSD 4.4 also does reset.
6857	*/
6858	if (sk->sk_shutdown & RCV_SHUTDOWN) {
6859	if (TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq &&
6860	after(TCP_SKB_CB(skb)->end_seq - th->fin, tp->rcv_nxt)) {
6861	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
6862	tcp_reset(sk, skb);
6863	return SKB_DROP_REASON_TCP_ABORT_ON_DATA;
6864	}
6865	}
6866	fallthrough;
6867	case TCP_ESTABLISHED:
6868	tcp_data_queue(sk, skb);
6869	queued = 1;
6870	break;
6871	}
6872
6873	/* tcp_data could move socket to TIME-WAIT */
6874	if (sk->sk_state != TCP_CLOSE) {
6875	tcp_data_snd_check(sk);
6876	tcp_ack_snd_check(sk);
6877	}
6878
6879	if (!queued) {
6880	discard:
6881	tcp_drop_reason(sk, skb, reason);
6882	}
6883	return 0;
6884
6885	consume:
6886	__kfree_skb(skb);
6887	return 0;
6888	}
6889	EXPORT_SYMBOL(tcp_rcv_state_process);
6890
6891	static inline void pr_drop_req(struct request_sock *req, __u16 port, int family)
6892	{
6893	struct inet_request_sock *ireq = inet_rsk(sk: req);
6894
6895	if (family == AF_INET)
6896	net_dbg_ratelimited("drop open request from %pI4/%u\n",
6897	&ireq->ir_rmt_addr, port);
6898	#if IS_ENABLED(CONFIG_IPV6)
6899	else if (family == AF_INET6)
6900	net_dbg_ratelimited("drop open request from %pI6/%u\n",
6901	&ireq->ir_v6_rmt_addr, port);
6902	#endif
6903	}
6904
6905	/* RFC3168 : 6.1.1 SYN packets must not have ECT/ECN bits set
6906	*
6907	* If we receive a SYN packet with these bits set, it means a
6908	* network is playing bad games with TOS bits. In order to
6909	* avoid possible false congestion notifications, we disable
6910	* TCP ECN negotiation.
6911	*
6912	* Exception: tcp_ca wants ECN. This is required for DCTCP
6913	* congestion control: Linux DCTCP asserts ECT on all packets,
6914	* including SYN, which is most optimal solution; however,
6915	* others, such as FreeBSD do not.
6916	*
6917	* Exception: At least one of the reserved bits of the TCP header (th->res1) is
6918	* set, indicating the use of a future TCP extension (such as AccECN). See
6919	* RFC8311 §4.3 which updates RFC3168 to allow the development of such
6920	* extensions.
6921	*/
6922	static void tcp_ecn_create_request(struct request_sock *req,
6923	const struct sk_buff *skb,
6924	const struct sock *listen_sk,
6925	const struct dst_entry *dst)
6926	{
6927	const struct tcphdr *th = tcp_hdr(skb);
6928	const struct net *net = sock_net(sk: listen_sk);
6929	bool th_ecn = th->ece && th->cwr;
6930	bool ect, ecn_ok;
6931	u32 ecn_ok_dst;
6932
6933	if (!th_ecn)
6934	return;
6935
6936	ect = !INET_ECN_is_not_ect(TCP_SKB_CB(skb)->ip_dsfield);
6937	ecn_ok_dst = dst_feature(dst, DST_FEATURE_ECN_MASK);
6938	ecn_ok = READ_ONCE(net->ipv4.sysctl_tcp_ecn) || ecn_ok_dst;
6939
6940	if (((!ect || th->res1) && ecn_ok) || tcp_ca_needs_ecn(sk: listen_sk) ||
6941	(ecn_ok_dst & DST_FEATURE_ECN_CA) ||
6942	tcp_bpf_ca_needs_ecn(sk: (struct sock *)req))
6943	inet_rsk(sk: req)->ecn_ok = 1;
6944	}
6945
6946	static void tcp_openreq_init(struct request_sock *req,
6947	const struct tcp_options_received *rx_opt,
6948	struct sk_buff *skb, const struct sock *sk)
6949	{
6950	struct inet_request_sock *ireq = inet_rsk(sk: req);
6951
6952	req->rsk_rcv_wnd = 0; /* So that tcp_send_synack() knows! */
6953	tcp_rsk(req)->rcv_isn = TCP_SKB_CB(skb)->seq;
6954	tcp_rsk(req)->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
6955	tcp_rsk(req)->snt_synack = 0;
6956	tcp_rsk(req)->last_oow_ack_time = 0;
6957	req->mss = rx_opt->mss_clamp;
6958	req->ts_recent = rx_opt->saw_tstamp ? rx_opt->rcv_tsval : 0;
6959	ireq->tstamp_ok = rx_opt->tstamp_ok;
6960	ireq->sack_ok = rx_opt->sack_ok;
6961	ireq->snd_wscale = rx_opt->snd_wscale;
6962	ireq->wscale_ok = rx_opt->wscale_ok;
6963	ireq->acked = 0;
6964	ireq->ecn_ok = 0;
6965	ireq->ir_rmt_port = tcp_hdr(skb)->source;
6966	ireq->ir_num = ntohs(tcp_hdr(skb)->dest);
6967	ireq->ir_mark = inet_request_mark(sk, skb);
6968	#if IS_ENABLED(CONFIG_SMC)
6969	ireq->smc_ok = rx_opt->smc_ok && !(tcp_sk(sk)->smc_hs_congested &&
6970	tcp_sk(sk)->smc_hs_congested(sk));
6971	#endif
6972	}
6973
6974	struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops,
6975	struct sock *sk_listener,
6976	bool attach_listener)
6977	{
6978	struct request_sock *req = reqsk_alloc(ops, sk_listener,
6979	attach_listener);
6980
6981	if (req) {
6982	struct inet_request_sock *ireq = inet_rsk(sk: req);
6983
6984	ireq->ireq_opt = NULL;
6985	#if IS_ENABLED(CONFIG_IPV6)
6986	ireq->pktopts = NULL;
6987	#endif
6988	atomic64_set(v: &ireq->ir_cookie, i: 0);
6989	ireq->ireq_state = TCP_NEW_SYN_RECV;
6990	write_pnet(pnet: &ireq->ireq_net, net: sock_net(sk: sk_listener));
6991	ireq->ireq_family = sk_listener->sk_family;
6992	req->timeout = TCP_TIMEOUT_INIT;
6993	}
6994
6995	return req;
6996	}
6997	EXPORT_SYMBOL(inet_reqsk_alloc);
6998
6999	/*
7000	* Return true if a syncookie should be sent
7001	*/
7002	static bool tcp_syn_flood_action(const struct sock *sk, const char *proto)
7003	{
7004	struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
7005	const char *msg = "Dropping request";
7006	struct net *net = sock_net(sk);
7007	bool want_cookie = false;
7008	u8 syncookies;
7009
7010	syncookies = READ_ONCE(net->ipv4.sysctl_tcp_syncookies);
7011
7012	#ifdef CONFIG_SYN_COOKIES
7013	if (syncookies) {
7014	msg = "Sending cookies";
7015	want_cookie = true;
7016	__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDOCOOKIES);
7017	} else
7018	#endif
7019	__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPREQQFULLDROP);
7020
7021	if (!READ_ONCE(queue->synflood_warned) && syncookies != 2 &&
7022	xchg(&queue->synflood_warned, 1) == 0) {
7023	if (IS_ENABLED(CONFIG_IPV6) && sk->sk_family == AF_INET6) {
7024	net_info_ratelimited("%s: Possible SYN flooding on port [%pI6c]:%u. %s.\n",
7025	proto, inet6_rcv_saddr(sk),
7026	sk->sk_num, msg);
7027	} else {
7028	net_info_ratelimited("%s: Possible SYN flooding on port %pI4:%u. %s.\n",
7029	proto, &sk->sk_rcv_saddr,
7030	sk->sk_num, msg);
7031	}
7032	}
7033
7034	return want_cookie;
7035	}
7036
7037	static void tcp_reqsk_record_syn(const struct sock *sk,
7038	struct request_sock *req,
7039	const struct sk_buff *skb)
7040	{
7041	if (tcp_sk(sk)->save_syn) {
7042	u32 len = skb_network_header_len(skb) + tcp_hdrlen(skb);
7043	struct saved_syn *saved_syn;
7044	u32 mac_hdrlen;
7045	void *base;
7046
7047	if (tcp_sk(sk)->save_syn == 2) { /* Save full header. */
7048	base = skb_mac_header(skb);
7049	mac_hdrlen = skb_mac_header_len(skb);
7050	len += mac_hdrlen;
7051	} else {
7052	base = skb_network_header(skb);
7053	mac_hdrlen = 0;
7054	}
7055
7056	saved_syn = kmalloc(struct_size(saved_syn, data, len),
7057	GFP_ATOMIC);
7058	if (saved_syn) {
7059	saved_syn->mac_hdrlen = mac_hdrlen;
7060	saved_syn->network_hdrlen = skb_network_header_len(skb);
7061	saved_syn->tcp_hdrlen = tcp_hdrlen(skb);
7062	memcpy(saved_syn->data, base, len);
7063	req->saved_syn = saved_syn;
7064	}
7065	}
7066	}
7067
7068	/* If a SYN cookie is required and supported, returns a clamped MSS value to be
7069	* used for SYN cookie generation.
7070	*/
7071	u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
7072	const struct tcp_request_sock_ops *af_ops,
7073	struct sock *sk, struct tcphdr *th)
7074	{
7075	struct tcp_sock *tp = tcp_sk(sk);
7076	u16 mss;
7077
7078	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies) != 2 &&
7079	!inet_csk_reqsk_queue_is_full(sk))
7080	return 0;
7081
7082	if (!tcp_syn_flood_action(sk, proto: rsk_ops->slab_name))
7083	return 0;
7084
7085	if (sk_acceptq_is_full(sk)) {
7086	NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
7087	return 0;
7088	}
7089
7090	mss = tcp_parse_mss_option(th, tp->rx_opt.user_mss);
7091	if (!mss)
7092	mss = af_ops->mss_clamp;
7093
7094	return mss;
7095	}
7096	EXPORT_SYMBOL_GPL(tcp_get_syncookie_mss);
7097
7098	int tcp_conn_request(struct request_sock_ops *rsk_ops,
7099	const struct tcp_request_sock_ops *af_ops,
7100	struct sock *sk, struct sk_buff *skb)
7101	{
7102	struct tcp_fastopen_cookie foc = { .len = -1 };
7103	__u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn;
7104	struct tcp_options_received tmp_opt;
7105	struct tcp_sock *tp = tcp_sk(sk);
7106	struct net *net = sock_net(sk);
7107	struct sock *fastopen_sk = NULL;
7108	struct request_sock *req;
7109	bool want_cookie = false;
7110	struct dst_entry *dst;
7111	struct flowi fl;
7112	u8 syncookies;
7113
7114	#ifdef CONFIG_TCP_AO
7115	const struct tcp_ao_hdr *aoh;
7116	#endif
7117
7118	syncookies = READ_ONCE(net->ipv4.sysctl_tcp_syncookies);
7119
7120	/* TW buckets are converted to open requests without
7121	* limitations, they conserve resources and peer is
7122	* evidently real one.
7123	*/
7124	if ((syncookies == 2 || inet_csk_reqsk_queue_is_full(sk)) && !isn) {
7125	want_cookie = tcp_syn_flood_action(sk, proto: rsk_ops->slab_name);
7126	if (!want_cookie)
7127	goto drop;
7128	}
7129
7130	if (sk_acceptq_is_full(sk)) {
7131	NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
7132	goto drop;
7133	}
7134
7135	req = inet_reqsk_alloc(rsk_ops, sk, !want_cookie);
7136	if (!req)
7137	goto drop;
7138
7139	req->syncookie = want_cookie;
7140	tcp_rsk(req)->af_specific = af_ops;
7141	tcp_rsk(req)->ts_off = 0;
7142	tcp_rsk(req)->req_usec_ts = false;
7143	#if IS_ENABLED(CONFIG_MPTCP)
7144	tcp_rsk(req)->is_mptcp = 0;
7145	#endif
7146
7147	tcp_clear_options(rx_opt: &tmp_opt);
7148	tmp_opt.mss_clamp = af_ops->mss_clamp;
7149	tmp_opt.user_mss = tp->rx_opt.user_mss;
7150	tcp_parse_options(sock_net(sk), skb, &tmp_opt, 0,
7151	want_cookie ? NULL : &foc);
7152
7153	if (want_cookie && !tmp_opt.saw_tstamp)
7154	tcp_clear_options(rx_opt: &tmp_opt);
7155
7156	if (IS_ENABLED(CONFIG_SMC) && want_cookie)
7157	tmp_opt.smc_ok = 0;
7158
7159	tmp_opt.tstamp_ok = tmp_opt.saw_tstamp;
7160	tcp_openreq_init(req, rx_opt: &tmp_opt, skb, sk);
7161	inet_rsk(sk: req)->no_srccheck = inet_test_bit(TRANSPARENT, sk);
7162
7163	/* Note: tcp_v6_init_req() might override ir_iif for link locals */
7164	inet_rsk(sk: req)->ir_iif = inet_request_bound_dev_if(sk, skb);
7165
7166	dst = af_ops->route_req(sk, skb, &fl, req);
7167	if (!dst)
7168	goto drop_and_free;
7169
7170	if (tmp_opt.tstamp_ok) {
7171	tcp_rsk(req)->req_usec_ts = dst_tcp_usec_ts(dst);
7172	tcp_rsk(req)->ts_off = af_ops->init_ts_off(net, skb);
7173	}
7174	if (!want_cookie && !isn) {
7175	int max_syn_backlog = READ_ONCE(net->ipv4.sysctl_max_syn_backlog);
7176
7177	/* Kill the following clause, if you dislike this way. */
7178	if (!syncookies &&
7179	(max_syn_backlog - inet_csk_reqsk_queue_len(sk) <
7180	(max_syn_backlog >> 2)) &&
7181	!tcp_peer_is_proven(req, dst)) {
7182	/* Without syncookies last quarter of
7183	* backlog is filled with destinations,
7184	* proven to be alive.
7185	* It means that we continue to communicate
7186	* to destinations, already remembered
7187	* to the moment of synflood.
7188	*/
7189	pr_drop_req(req, ntohs(tcp_hdr(skb)->source),
7190	family: rsk_ops->family);
7191	goto drop_and_release;
7192	}
7193
7194	isn = af_ops->init_seq(skb);
7195	}
7196
7197	tcp_ecn_create_request(req, skb, listen_sk: sk, dst);
7198
7199	if (want_cookie) {
7200	isn = cookie_init_sequence(ops: af_ops, sk, skb, mss: &req->mss);
7201	if (!tmp_opt.tstamp_ok)
7202	inet_rsk(sk: req)->ecn_ok = 0;
7203	}
7204
7205	#ifdef CONFIG_TCP_AO
7206	if (tcp_parse_auth_options(th: tcp_hdr(skb), NULL, aoh: &aoh))
7207	goto drop_and_release; /* Invalid TCP options */
7208	if (aoh) {
7209	tcp_rsk(req)->used_tcp_ao = true;
7210	tcp_rsk(req)->ao_rcv_next = aoh->keyid;
7211	tcp_rsk(req)->ao_keyid = aoh->rnext_keyid;
7212
7213	} else {
7214	tcp_rsk(req)->used_tcp_ao = false;
7215	}
7216	#endif
7217	tcp_rsk(req)->snt_isn = isn;
7218	tcp_rsk(req)->txhash = net_tx_rndhash();
7219	tcp_rsk(req)->syn_tos = TCP_SKB_CB(skb)->ip_dsfield;
7220	tcp_openreq_init_rwin(req, sk_listener: sk, dst);
7221	sk_rx_queue_set(sk: req_to_sk(req), skb);
7222	if (!want_cookie) {
7223	tcp_reqsk_record_syn(sk, req, skb);
7224	fastopen_sk = tcp_try_fastopen(sk, skb, req, foc: &foc, dst);
7225	}
7226	if (fastopen_sk) {
7227	af_ops->send_synack(fastopen_sk, dst, &fl, req,
7228	&foc, TCP_SYNACK_FASTOPEN, skb);
7229	/* Add the child socket directly into the accept queue */
7230	if (!inet_csk_reqsk_queue_add(sk, req, child: fastopen_sk)) {
7231	reqsk_fastopen_remove(sk: fastopen_sk, req, reset: false);
7232	bh_unlock_sock(fastopen_sk);
7233	sock_put(sk: fastopen_sk);
7234	goto drop_and_free;
7235	}
7236	sk->sk_data_ready(sk);
7237	bh_unlock_sock(fastopen_sk);
7238	sock_put(sk: fastopen_sk);
7239	} else {
7240	tcp_rsk(req)->tfo_listener = false;
7241	if (!want_cookie) {
7242	req->timeout = tcp_timeout_init(sk: (struct sock *)req);
7243	inet_csk_reqsk_queue_hash_add(sk, req, timeout: req->timeout);
7244	}
7245	af_ops->send_synack(sk, dst, &fl, req, &foc,
7246	!want_cookie ? TCP_SYNACK_NORMAL :
7247	TCP_SYNACK_COOKIE,
7248	skb);
7249	if (want_cookie) {
7250	reqsk_free(req);
7251	return 0;
7252	}
7253	}
7254	reqsk_put(req);
7255	return 0;
7256
7257	drop_and_release:
7258	dst_release(dst);
7259	drop_and_free:
7260	__reqsk_free(req);
7261	drop:
7262	tcp_listendrop(sk);
7263	return 0;
7264	}
7265	EXPORT_SYMBOL(tcp_conn_request);
7266