1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* INET An implementation of the TCP/IP protocol suite for the LINUX
4	* operating system. INET is implemented using the BSD Socket
5	* interface as the means of communication with the user level.
6	*
7	* Implementation of the Transmission Control Protocol(TCP).
8	*
9	* Authors: Ross Biro
10	* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11	* Mark Evans, <evansmp@uhura.aston.ac.uk>
12	* Corey Minyard <wf-rch!minyard@relay.EU.net>
13	* Florian La Roche, <flla@stud.uni-sb.de>
14	* Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15	* Linus Torvalds, <torvalds@cs.helsinki.fi>
16	* Alan Cox, <gw4pts@gw4pts.ampr.org>
17	* Matthew Dillon, <dillon@apollo.west.oic.com>
18	* Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19	* Jorge Cwik, <jorge@laser.satlink.net>
20	*
21	* Fixes:
22	* Alan Cox : Numerous verify_area() calls
23	* Alan Cox : Set the ACK bit on a reset
24	* Alan Cox : Stopped it crashing if it closed while
25	* sk->inuse=1 and was trying to connect
26	* (tcp_err()).
27	* Alan Cox : All icmp error handling was broken
28	* pointers passed where wrong and the
29	* socket was looked up backwards. Nobody
30	* tested any icmp error code obviously.
31	* Alan Cox : tcp_err() now handled properly. It
32	* wakes people on errors. poll
33	* behaves and the icmp error race
34	* has gone by moving it into sock.c
35	* Alan Cox : tcp_send_reset() fixed to work for
36	* everything not just packets for
37	* unknown sockets.
38	* Alan Cox : tcp option processing.
39	* Alan Cox : Reset tweaked (still not 100%) [Had
40	* syn rule wrong]
41	* Herp Rosmanith : More reset fixes
42	* Alan Cox : No longer acks invalid rst frames.
43	* Acking any kind of RST is right out.
44	* Alan Cox : Sets an ignore me flag on an rst
45	* receive otherwise odd bits of prattle
46	* escape still
47	* Alan Cox : Fixed another acking RST frame bug.
48	* Should stop LAN workplace lockups.
49	* Alan Cox : Some tidyups using the new skb list
50	* facilities
51	* Alan Cox : sk->keepopen now seems to work
52	* Alan Cox : Pulls options out correctly on accepts
53	* Alan Cox : Fixed assorted sk->rqueue->next errors
54	* Alan Cox : PSH doesn't end a TCP read. Switched a
55	* bit to skb ops.
56	* Alan Cox : Tidied tcp_data to avoid a potential
57	* nasty.
58	* Alan Cox : Added some better commenting, as the
59	* tcp is hard to follow
60	* Alan Cox : Removed incorrect check for 20 * psh
61	* Michael O'Reilly : ack < copied bug fix.
62	* Johannes Stille : Misc tcp fixes (not all in yet).
63	* Alan Cox : FIN with no memory -> CRASH
64	* Alan Cox : Added socket option proto entries.
65	* Also added awareness of them to accept.
66	* Alan Cox : Added TCP options (SOL_TCP)
67	* Alan Cox : Switched wakeup calls to callbacks,
68	* so the kernel can layer network
69	* sockets.
70	* Alan Cox : Use ip_tos/ip_ttl settings.
71	* Alan Cox : Handle FIN (more) properly (we hope).
72	* Alan Cox : RST frames sent on unsynchronised
73	* state ack error.
74	* Alan Cox : Put in missing check for SYN bit.
75	* Alan Cox : Added tcp_select_window() aka NET2E
76	* window non shrink trick.
77	* Alan Cox : Added a couple of small NET2E timer
78	* fixes
79	* Charles Hedrick : TCP fixes
80	* Toomas Tamm : TCP window fixes
81	* Alan Cox : Small URG fix to rlogin ^C ack fight
82	* Charles Hedrick : Rewrote most of it to actually work
83	* Linus : Rewrote tcp_read() and URG handling
84	* completely
85	* Gerhard Koerting: Fixed some missing timer handling
86	* Matthew Dillon : Reworked TCP machine states as per RFC
87	* Gerhard Koerting: PC/TCP workarounds
88	* Adam Caldwell : Assorted timer/timing errors
89	* Matthew Dillon : Fixed another RST bug
90	* Alan Cox : Move to kernel side addressing changes.
91	* Alan Cox : Beginning work on TCP fastpathing
92	* (not yet usable)
93	* Arnt Gulbrandsen: Turbocharged tcp_check() routine.
94	* Alan Cox : TCP fast path debugging
95	* Alan Cox : Window clamping
96	* Michael Riepe : Bug in tcp_check()
97	* Matt Dillon : More TCP improvements and RST bug fixes
98	* Matt Dillon : Yet more small nasties remove from the
99	* TCP code (Be very nice to this man if
100	* tcp finally works 100%) 8)
101	* Alan Cox : BSD accept semantics.
102	* Alan Cox : Reset on closedown bug.
103	* Peter De Schrijver : ENOTCONN check missing in tcp_sendto().
104	* Michael Pall : Handle poll() after URG properly in
105	* all cases.
106	* Michael Pall : Undo the last fix in tcp_read_urg()
107	* (multi URG PUSH broke rlogin).
108	* Michael Pall : Fix the multi URG PUSH problem in
109	* tcp_readable(), poll() after URG
110	* works now.
111	* Michael Pall : recv(...,MSG_OOB) never blocks in the
112	* BSD api.
113	* Alan Cox : Changed the semantics of sk->socket to
114	* fix a race and a signal problem with
115	* accept() and async I/O.
116	* Alan Cox : Relaxed the rules on tcp_sendto().
117	* Yury Shevchuk : Really fixed accept() blocking problem.
118	* Craig I. Hagan : Allow for BSD compatible TIME_WAIT for
119	* clients/servers which listen in on
120	* fixed ports.
121	* Alan Cox : Cleaned the above up and shrank it to
122	* a sensible code size.
123	* Alan Cox : Self connect lockup fix.
124	* Alan Cox : No connect to multicast.
125	* Ross Biro : Close unaccepted children on master
126	* socket close.
127	* Alan Cox : Reset tracing code.
128	* Alan Cox : Spurious resets on shutdown.
129	* Alan Cox : Giant 15 minute/60 second timer error
130	* Alan Cox : Small whoops in polling before an
131	* accept.
132	* Alan Cox : Kept the state trace facility since
133	* it's handy for debugging.
134	* Alan Cox : More reset handler fixes.
135	* Alan Cox : Started rewriting the code based on
136	* the RFC's for other useful protocol
137	* references see: Comer, KA9Q NOS, and
138	* for a reference on the difference
139	* between specifications and how BSD
140	* works see the 4.4lite source.
141	* A.N.Kuznetsov : Don't time wait on completion of tidy
142	* close.
143	* Linus Torvalds : Fin/Shutdown & copied_seq changes.
144	* Linus Torvalds : Fixed BSD port reuse to work first syn
145	* Alan Cox : Reimplemented timers as per the RFC
146	* and using multiple timers for sanity.
147	* Alan Cox : Small bug fixes, and a lot of new
148	* comments.
149	* Alan Cox : Fixed dual reader crash by locking
150	* the buffers (much like datagram.c)
151	* Alan Cox : Fixed stuck sockets in probe. A probe
152	* now gets fed up of retrying without
153	* (even a no space) answer.
154	* Alan Cox : Extracted closing code better
155	* Alan Cox : Fixed the closing state machine to
156	* resemble the RFC.
157	* Alan Cox : More 'per spec' fixes.
158	* Jorge Cwik : Even faster checksumming.
159	* Alan Cox : tcp_data() doesn't ack illegal PSH
160	* only frames. At least one pc tcp stack
161	* generates them.
162	* Alan Cox : Cache last socket.
163	* Alan Cox : Per route irtt.
164	* Matt Day : poll()->select() match BSD precisely on error
165	* Alan Cox : New buffers
166	* Marc Tamsky : Various sk->prot->retransmits and
167	* sk->retransmits misupdating fixed.
168	* Fixed tcp_write_timeout: stuck close,
169	* and TCP syn retries gets used now.
170	* Mark Yarvis : In tcp_read_wakeup(), don't send an
171	* ack if state is TCP_CLOSED.
172	* Alan Cox : Look up device on a retransmit - routes may
173	* change. Doesn't yet cope with MSS shrink right
174	* but it's a start!
175	* Marc Tamsky : Closing in closing fixes.
176	* Mike Shaver : RFC1122 verifications.
177	* Alan Cox : rcv_saddr errors.
178	* Alan Cox : Block double connect().
179	* Alan Cox : Small hooks for enSKIP.
180	* Alexey Kuznetsov: Path MTU discovery.
181	* Alan Cox : Support soft errors.
182	* Alan Cox : Fix MTU discovery pathological case
183	* when the remote claims no mtu!
184	* Marc Tamsky : TCP_CLOSE fix.
185	* Colin (G3TNE) : Send a reset on syn ack replies in
186	* window but wrong (fixes NT lpd problems)
187	* Pedro Roque : Better TCP window handling, delayed ack.
188	* Joerg Reuter : No modification of locked buffers in
189	* tcp_do_retransmit()
190	* Eric Schenk : Changed receiver side silly window
191	* avoidance algorithm to BSD style
192	* algorithm. This doubles throughput
193	* against machines running Solaris,
194	* and seems to result in general
195	* improvement.
196	* Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD
197	* Willy Konynenberg : Transparent proxying support.
198	* Mike McLagan : Routing by source
199	* Keith Owens : Do proper merging with partial SKB's in
200	* tcp_do_sendmsg to avoid burstiness.
201	* Eric Schenk : Fix fast close down bug with
202	* shutdown() followed by close().
203	* Andi Kleen : Make poll agree with SIGIO
204	* Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and
205	* lingertime == 0 (RFC 793 ABORT Call)
206	* Hirokazu Takahashi : Use copy_from_user() instead of
207	* csum_and_copy_from_user() if possible.
208	*
209	* Description of States:
210	*
211	* TCP_SYN_SENT sent a connection request, waiting for ack
212	*
213	* TCP_SYN_RECV received a connection request, sent ack,
214	* waiting for final ack in three-way handshake.
215	*
216	* TCP_ESTABLISHED connection established
217	*
218	* TCP_FIN_WAIT1 our side has shutdown, waiting to complete
219	* transmission of remaining buffered data
220	*
221	* TCP_FIN_WAIT2 all buffered data sent, waiting for remote
222	* to shutdown
223	*
224	* TCP_CLOSING both sides have shutdown but we still have
225	* data we have to finish sending
226	*
227	* TCP_TIME_WAIT timeout to catch resent junk before entering
228	* closed, can only be entered from FIN_WAIT2
229	* or CLOSING. Required because the other end
230	* may not have gotten our last ACK causing it
231	* to retransmit the data packet (which we ignore)
232	*
233	* TCP_CLOSE_WAIT remote side has shutdown and is waiting for
234	* us to finish writing our data and to shutdown
235	* (we have to close() to move on to LAST_ACK)
236	*
237	* TCP_LAST_ACK out side has shutdown after remote has
238	* shutdown. There may still be data in our
239	* buffer that we have to finish sending
240	*
241	* TCP_CLOSE socket is finished
242	*/
243
244	#define pr_fmt(fmt) "TCP: " fmt
245
246	#include <crypto/hash.h>
247	#include <linux/kernel.h>
248	#include <linux/module.h>
249	#include <linux/types.h>
250	#include <linux/fcntl.h>
251	#include <linux/poll.h>
252	#include <linux/inet_diag.h>
253	#include <linux/init.h>
254	#include <linux/fs.h>
255	#include <linux/skbuff.h>
256	#include <linux/scatterlist.h>
257	#include <linux/splice.h>
258	#include <linux/net.h>
259	#include <linux/socket.h>
260	#include <linux/random.h>
261	#include <linux/memblock.h>
262	#include <linux/highmem.h>
263	#include <linux/cache.h>
264	#include <linux/err.h>
265	#include <linux/time.h>
266	#include <linux/slab.h>
267	#include <linux/errqueue.h>
268	#include <linux/static_key.h>
269	#include <linux/btf.h>
270
271	#include <net/icmp.h>
272	#include <net/inet_common.h>
273	#include <net/tcp.h>
274	#include <net/mptcp.h>
275	#include <net/proto_memory.h>
276	#include <net/xfrm.h>
277	#include <net/ip.h>
278	#include <net/sock.h>
279	#include <net/rstreason.h>
280
281	#include <linux/uaccess.h>
282	#include <asm/ioctls.h>
283	#include <net/busy_poll.h>
284	#include <net/hotdata.h>
285	#include <trace/events/tcp.h>
286	#include <net/rps.h>
287
288	#include "../core/devmem.h"
289
290	/* Track pending CMSGs. */
291	enum {
292	TCP_CMSG_INQ = 1,
293	TCP_CMSG_TS = 2
294	};
295
296	DEFINE_PER_CPU(unsigned int, tcp_orphan_count);
297	EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count);
298
299	DEFINE_PER_CPU(u32, tcp_tw_isn);
300	EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn);
301
302	long sysctl_tcp_mem[3] __read_mostly;
303	EXPORT_IPV6_MOD(sysctl_tcp_mem);
304
305	atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp; /* Current allocated memory. */
306	EXPORT_IPV6_MOD(tcp_memory_allocated);
307	DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
308	EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc);
309
310	#if IS_ENABLED(CONFIG_SMC)
311	DEFINE_STATIC_KEY_FALSE(tcp_have_smc);
312	EXPORT_SYMBOL(tcp_have_smc);
313	#endif
314
315	/*
316	* Current number of TCP sockets.
317	*/
318	struct percpu_counter tcp_sockets_allocated ____cacheline_aligned_in_smp;
319	EXPORT_IPV6_MOD(tcp_sockets_allocated);
320
321	/*
322	* TCP splice context
323	*/
324	struct tcp_splice_state {
325	struct pipe_inode_info *pipe;
326	size_t len;
327	unsigned int flags;
328	};
329
330	/*
331	* Pressure flag: try to collapse.
332	* Technical note: it is used by multiple contexts non atomically.
333	* All the __sk_mem_schedule() is of this nature: accounting
334	* is strict, actions are advisory and have some latency.
335	*/
336	unsigned long tcp_memory_pressure __read_mostly;
337	EXPORT_SYMBOL_GPL(tcp_memory_pressure);
338
339	void tcp_enter_memory_pressure(struct sock *sk)
340	{
341	unsigned long val;
342
343	if (READ_ONCE(tcp_memory_pressure))
344	return;
345	val = jiffies;
346
347	if (!val)
348	val--;
349	if (!cmpxchg(&tcp_memory_pressure, 0, val))
350	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
351	}
352	EXPORT_IPV6_MOD_GPL(tcp_enter_memory_pressure);
353
354	void tcp_leave_memory_pressure(struct sock *sk)
355	{
356	unsigned long val;
357
358	if (!READ_ONCE(tcp_memory_pressure))
359	return;
360	val = xchg(&tcp_memory_pressure, 0);
361	if (val)
362	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO,
363	jiffies_to_msecs(jiffies - val));
364	}
365	EXPORT_IPV6_MOD_GPL(tcp_leave_memory_pressure);
366
367	/* Convert seconds to retransmits based on initial and max timeout */
368	static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
369	{
370	u8 res = 0;
371
372	if (seconds > 0) {
373	int period = timeout;
374
375	res = 1;
376	while (seconds > period && res < 255) {
377	res++;
378	timeout <<= 1;
379	if (timeout > rto_max)
380	timeout = rto_max;
381	period += timeout;
382	}
383	}
384	return res;
385	}
386
387	/* Convert retransmits to seconds based on initial and max timeout */
388	static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
389	{
390	int period = 0;
391
392	if (retrans > 0) {
393	period = timeout;
394	while (--retrans) {
395	timeout <<= 1;
396	if (timeout > rto_max)
397	timeout = rto_max;
398	period += timeout;
399	}
400	}
401	return period;
402	}
403
404	static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp)
405	{
406	u32 rate = READ_ONCE(tp->rate_delivered);
407	u32 intv = READ_ONCE(tp->rate_interval_us);
408	u64 rate64 = 0;
409
410	if (rate && intv) {
411	rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;
412	do_div(rate64, intv);
413	}
414	return rate64;
415	}
416
417	/* Address-family independent initialization for a tcp_sock.
418	*
419	* NOTE: A lot of things set to zero explicitly by call to
420	* sk_alloc() so need not be done here.
421	*/
422	void tcp_init_sock(struct sock *sk)
423	{
424	struct inet_connection_sock *icsk = inet_csk(sk);
425	struct tcp_sock *tp = tcp_sk(sk);
426	int rto_min_us, rto_max_ms;
427
428	tp->out_of_order_queue = RB_ROOT;
429	sk->tcp_rtx_queue = RB_ROOT;
430	tcp_init_xmit_timers(sk);
431	INIT_LIST_HEAD(list: &tp->tsq_node);
432	INIT_LIST_HEAD(list: &tp->tsorted_sent_queue);
433
434	icsk->icsk_rto = TCP_TIMEOUT_INIT;
435
436	rto_max_ms = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_max_ms);
437	icsk->icsk_rto_max = msecs_to_jiffies(m: rto_max_ms);
438
439	rto_min_us = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_min_us);
440	icsk->icsk_rto_min = usecs_to_jiffies(u: rto_min_us);
441	icsk->icsk_delack_max = TCP_DELACK_MAX;
442	tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
443	minmax_reset(m: &tp->rtt_min, tcp_jiffies32, meas: ~0U);
444
445	/* So many TCP implementations out there (incorrectly) count the
446	* initial SYN frame in their delayed-ACK and congestion control
447	* algorithms that we must have the following bandaid to talk
448	* efficiently to them. -DaveM
449	*/
450	tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
451
452	/* There's a bubble in the pipe until at least the first ACK. */
453	tp->app_limited = ~0U;
454	tp->rate_app_limited = 1;
455
456	/* See draft-stevens-tcpca-spec-01 for discussion of the
457	* initialization of these values.
458	*/
459	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
460	tp->snd_cwnd_clamp = ~0;
461	tp->mss_cache = TCP_MSS_DEFAULT;
462
463	tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering);
464	tcp_assign_congestion_control(sk);
465
466	tp->tsoffset = 0;
467	tp->rack.reo_wnd_steps = 1;
468
469	sk->sk_write_space = sk_stream_write_space;
470	sock_set_flag(sk, flag: SOCK_USE_WRITE_QUEUE);
471
472	icsk->icsk_sync_mss = tcp_sync_mss;
473
474	WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1]));
475	WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1]));
476	tcp_scaling_ratio_init(sk);
477
478	set_bit(nr: SOCK_SUPPORT_ZC, addr: &sk->sk_socket->flags);
479	sk_sockets_allocated_inc(sk);
480	xa_init_flags(xa: &sk->sk_user_frags, XA_FLAGS_ALLOC1);
481	}
482	EXPORT_IPV6_MOD(tcp_init_sock);
483
484	static void tcp_tx_timestamp(struct sock *sk, struct sockcm_cookie *sockc)
485	{
486	struct sk_buff *skb = tcp_write_queue_tail(sk);
487	u32 tsflags = sockc->tsflags;
488
489	if (tsflags && skb) {
490	struct skb_shared_info *shinfo = skb_shinfo(skb);
491	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
492
493	sock_tx_timestamp(sk, sockc, tx_flags: &shinfo->tx_flags);
494	if (tsflags & SOF_TIMESTAMPING_TX_ACK)
495	tcb->txstamp_ack |= TSTAMP_ACK_SK;
496	if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
497	shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
498	}
499
500	if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) &&
501	SK_BPF_CB_FLAG_TEST(sk, SK_BPF_CB_TX_TIMESTAMPING) && skb)
502	bpf_skops_tx_timestamping(sk, skb, op: BPF_SOCK_OPS_TSTAMP_SENDMSG_CB);
503	}
504
505	static bool tcp_stream_is_readable(struct sock *sk, int target)
506	{
507	if (tcp_epollin_ready(sk, target))
508	return true;
509	return sk_is_readable(sk);
510	}
511
512	/*
513	* Wait for a TCP event.
514	*
515	* Note that we don't need to lock the socket, as the upper poll layers
516	* take care of normal races (between the test and the event) and we don't
517	* go look at any of the socket buffers directly.
518	*/
519	__poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
520	{
521	__poll_t mask;
522	struct sock *sk = sock->sk;
523	const struct tcp_sock *tp = tcp_sk(sk);
524	u8 shutdown;
525	int state;
526
527	sock_poll_wait(filp: file, sock, p: wait);
528
529	state = inet_sk_state_load(sk);
530	if (state == TCP_LISTEN)
531	return inet_csk_listen_poll(sk);
532
533	/* Socket is not locked. We are protected from async events
534	* by poll logic and correct handling of state changes
535	* made by other threads is impossible in any case.
536	*/
537
538	mask = 0;
539
540	/*
541	* EPOLLHUP is certainly not done right. But poll() doesn't
542	* have a notion of HUP in just one direction, and for a
543	* socket the read side is more interesting.
544	*
545	* Some poll() documentation says that EPOLLHUP is incompatible
546	* with the EPOLLOUT/POLLWR flags, so somebody should check this
547	* all. But careful, it tends to be safer to return too many
548	* bits than too few, and you can easily break real applications
549	* if you don't tell them that something has hung up!
550	*
551	* Check-me.
552	*
553	* Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and
554	* our fs/select.c). It means that after we received EOF,
555	* poll always returns immediately, making impossible poll() on write()
556	* in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP
557	* if and only if shutdown has been made in both directions.
558	* Actually, it is interesting to look how Solaris and DUX
559	* solve this dilemma. I would prefer, if EPOLLHUP were maskable,
560	* then we could set it on SND_SHUTDOWN. BTW examples given
561	* in Stevens' books assume exactly this behaviour, it explains
562	* why EPOLLHUP is incompatible with EPOLLOUT. --ANK
563	*
564	* NOTE. Check for TCP_CLOSE is added. The goal is to prevent
565	* blocking on fresh not-connected or disconnected socket. --ANK
566	*/
567	shutdown = READ_ONCE(sk->sk_shutdown);
568	if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
569	mask |= EPOLLHUP;
570	if (shutdown & RCV_SHUTDOWN)
571	mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
572
573	/* Connected or passive Fast Open socket? */
574	if (state != TCP_SYN_SENT &&
575	(state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {
576	int target = sock_rcvlowat(sk, waitall: 0, INT_MAX);
577	u16 urg_data = READ_ONCE(tp->urg_data);
578
579	if (unlikely(urg_data) &&
580	READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&
581	!sock_flag(sk, flag: SOCK_URGINLINE))
582	target++;
583
584	if (tcp_stream_is_readable(sk, target))
585	mask |= EPOLLIN | EPOLLRDNORM;
586
587	if (!(shutdown & SEND_SHUTDOWN)) {
588	if (__sk_stream_is_writeable(sk, wake: 1)) {
589	mask |= EPOLLOUT | EPOLLWRNORM;
590	} else { /* send SIGIO later */
591	sk_set_bit(nr: SOCKWQ_ASYNC_NOSPACE, sk);
592	set_bit(nr: SOCK_NOSPACE, addr: &sk->sk_socket->flags);
593
594	/* Race breaker. If space is freed after
595	* wspace test but before the flags are set,
596	* IO signal will be lost. Memory barrier
597	* pairs with the input side.
598	*/
599	smp_mb__after_atomic();
600	if (__sk_stream_is_writeable(sk, wake: 1))
601	mask |= EPOLLOUT | EPOLLWRNORM;
602	}
603	} else
604	mask |= EPOLLOUT | EPOLLWRNORM;
605
606	if (urg_data & TCP_URG_VALID)
607	mask |= EPOLLPRI;
608	} else if (state == TCP_SYN_SENT &&
609	inet_test_bit(DEFER_CONNECT, sk)) {
610	/* Active TCP fastopen socket with defer_connect
611	* Return EPOLLOUT so application can call write()
612	* in order for kernel to generate SYN+data
613	*/
614	mask |= EPOLLOUT | EPOLLWRNORM;
615	}
616	/* This barrier is coupled with smp_wmb() in tcp_done_with_error() */
617	smp_rmb();
618	if (READ_ONCE(sk->sk_err) ||
619	!skb_queue_empty_lockless(list: &sk->sk_error_queue))
620	mask |= EPOLLERR;
621
622	return mask;
623	}
624	EXPORT_SYMBOL(tcp_poll);
625
626	int tcp_ioctl(struct sock *sk, int cmd, int *karg)
627	{
628	struct tcp_sock *tp = tcp_sk(sk);
629	int answ;
630	bool slow;
631
632	switch (cmd) {
633	case SIOCINQ:
634	if (sk->sk_state == TCP_LISTEN)
635	return -EINVAL;
636
637	slow = lock_sock_fast(sk);
638	answ = tcp_inq(sk);
639	unlock_sock_fast(sk, slow);
640	break;
641	case SIOCATMARK:
642	answ = READ_ONCE(tp->urg_data) &&
643	READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq);
644	break;
645	case SIOCOUTQ:
646	if (sk->sk_state == TCP_LISTEN)
647	return -EINVAL;
648
649	if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
650	answ = 0;
651	else
652	answ = READ_ONCE(tp->write_seq) - tp->snd_una;
653	break;
654	case SIOCOUTQNSD:
655	if (sk->sk_state == TCP_LISTEN)
656	return -EINVAL;
657
658	if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
659	answ = 0;
660	else
661	answ = READ_ONCE(tp->write_seq) -
662	READ_ONCE(tp->snd_nxt);
663	break;
664	default:
665	return -ENOIOCTLCMD;
666	}
667
668	*karg = answ;
669	return 0;
670	}
671	EXPORT_IPV6_MOD(tcp_ioctl);
672
673	void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
674	{
675	TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
676	tp->pushed_seq = tp->write_seq;
677	}
678
679	static inline bool forced_push(const struct tcp_sock *tp)
680	{
681	return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
682	}
683
684	void tcp_skb_entail(struct sock *sk, struct sk_buff *skb)
685	{
686	struct tcp_sock *tp = tcp_sk(sk);
687	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
688
689	tcb->seq = tcb->end_seq = tp->write_seq;
690	tcb->tcp_flags = TCPHDR_ACK;
691	__skb_header_release(skb);
692	tcp_add_write_queue_tail(sk, skb);
693	sk_wmem_queued_add(sk, val: skb->truesize);
694	sk_mem_charge(sk, size: skb->truesize);
695	if (tp->nonagle & TCP_NAGLE_PUSH)
696	tp->nonagle &= ~TCP_NAGLE_PUSH;
697
698	tcp_slow_start_after_idle_check(sk);
699	}
700
701	static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
702	{
703	if (flags & MSG_OOB)
704	tp->snd_up = tp->write_seq;
705	}
706
707	/* If a not yet filled skb is pushed, do not send it if
708	* we have data packets in Qdisc or NIC queues :
709	* Because TX completion will happen shortly, it gives a chance
710	* to coalesce future sendmsg() payload into this skb, without
711	* need for a timer, and with no latency trade off.
712	* As packets containing data payload have a bigger truesize
713	* than pure acks (dataless) packets, the last checks prevent
714	* autocorking if we only have an ACK in Qdisc/NIC queues,
715	* or if TX completion was delayed after we processed ACK packet.
716	*/
717	static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
718	int size_goal)
719	{
720	return skb->len < size_goal &&
721	READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_autocorking) &&
722	!tcp_rtx_queue_empty(sk) &&
723	refcount_read(r: &sk->sk_wmem_alloc) > skb->truesize &&
724	tcp_skb_can_collapse_to(skb);
725	}
726
727	void tcp_push(struct sock *sk, int flags, int mss_now,
728	int nonagle, int size_goal)
729	{
730	struct tcp_sock *tp = tcp_sk(sk);
731	struct sk_buff *skb;
732
733	skb = tcp_write_queue_tail(sk);
734	if (!skb)
735	return;
736	if (!(flags & MSG_MORE) || forced_push(tp))
737	tcp_mark_push(tp, skb);
738
739	tcp_mark_urg(tp, flags);
740
741	if (tcp_should_autocork(sk, skb, size_goal)) {
742
743	/* avoid atomic op if TSQ_THROTTLED bit is already set */
744	if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
745	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
746	set_bit(nr: TSQ_THROTTLED, addr: &sk->sk_tsq_flags);
747	smp_mb__after_atomic();
748	}
749	/* It is possible TX completion already happened
750	* before we set TSQ_THROTTLED.
751	*/
752	if (refcount_read(r: &sk->sk_wmem_alloc) > skb->truesize)
753	return;
754	}
755
756	if (flags & MSG_MORE)
757	nonagle = TCP_NAGLE_CORK;
758
759	__tcp_push_pending_frames(sk, cur_mss: mss_now, nonagle);
760	}
761
762	static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
763	unsigned int offset, size_t len)
764	{
765	struct tcp_splice_state *tss = rd_desc->arg.data;
766	int ret;
767
768	ret = skb_splice_bits(skb, sk: skb->sk, offset, pipe: tss->pipe,
769	min(rd_desc->count, len), flags: tss->flags);
770	if (ret > 0)
771	rd_desc->count -= ret;
772	return ret;
773	}
774
775	static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
776	{
777	/* Store TCP splice context information in read_descriptor_t. */
778	read_descriptor_t rd_desc = {
779	.arg.data = tss,
780	.count = tss->len,
781	};
782
783	return tcp_read_sock(sk, desc: &rd_desc, recv_actor: tcp_splice_data_recv);
784	}
785
786	/**
787	* tcp_splice_read - splice data from TCP socket to a pipe
788	* @sock: socket to splice from
789	* @ppos: position (not valid)
790	* @pipe: pipe to splice to
791	* @len: number of bytes to splice
792	* @flags: splice modifier flags
793	*
794	* Description:
795	* Will read pages from given socket and fill them into a pipe.
796	*
797	**/
798	ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
799	struct pipe_inode_info *pipe, size_t len,
800	unsigned int flags)
801	{
802	struct sock *sk = sock->sk;
803	struct tcp_splice_state tss = {
804	.pipe = pipe,
805	.len = len,
806	.flags = flags,
807	};
808	long timeo;
809	ssize_t spliced;
810	int ret;
811
812	sock_rps_record_flow(sk);
813	/*
814	* We can't seek on a socket input
815	*/
816	if (unlikely(*ppos))
817	return -ESPIPE;
818
819	ret = spliced = 0;
820
821	lock_sock(sk);
822
823	timeo = sock_rcvtimeo(sk, noblock: sock->file->f_flags & O_NONBLOCK);
824	while (tss.len) {
825	ret = __tcp_splice_read(sk, tss: &tss);
826	if (ret < 0)
827	break;
828	else if (!ret) {
829	if (spliced)
830	break;
831	if (sock_flag(sk, flag: SOCK_DONE))
832	break;
833	if (sk->sk_err) {
834	ret = sock_error(sk);
835	break;
836	}
837	if (sk->sk_shutdown & RCV_SHUTDOWN)
838	break;
839	if (sk->sk_state == TCP_CLOSE) {
840	/*
841	* This occurs when user tries to read
842	* from never connected socket.
843	*/
844	ret = -ENOTCONN;
845	break;
846	}
847	if (!timeo) {
848	ret = -EAGAIN;
849	break;
850	}
851	/* if __tcp_splice_read() got nothing while we have
852	* an skb in receive queue, we do not want to loop.
853	* This might happen with URG data.
854	*/
855	if (!skb_queue_empty(list: &sk->sk_receive_queue))
856	break;
857	ret = sk_wait_data(sk, timeo: &timeo, NULL);
858	if (ret < 0)
859	break;
860	if (signal_pending(current)) {
861	ret = sock_intr_errno(timeo);
862	break;
863	}
864	continue;
865	}
866	tss.len -= ret;
867	spliced += ret;
868
869	if (!tss.len || !timeo)
870	break;
871	release_sock(sk);
872	lock_sock(sk);
873
874	if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
875	(sk->sk_shutdown & RCV_SHUTDOWN) ||
876	signal_pending(current))
877	break;
878	}
879
880	release_sock(sk);
881
882	if (spliced)
883	return spliced;
884
885	return ret;
886	}
887	EXPORT_IPV6_MOD(tcp_splice_read);
888
889	struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
890	bool force_schedule)
891	{
892	struct sk_buff *skb;
893
894	skb = alloc_skb_fclone(MAX_TCP_HEADER, priority: gfp);
895	if (likely(skb)) {
896	bool mem_scheduled;
897
898	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
899	if (force_schedule) {
900	mem_scheduled = true;
901	sk_forced_mem_schedule(sk, size: skb->truesize);
902	} else {
903	mem_scheduled = sk_wmem_schedule(sk, size: skb->truesize);
904	}
905	if (likely(mem_scheduled)) {
906	skb_reserve(skb, MAX_TCP_HEADER);
907	skb->ip_summed = CHECKSUM_PARTIAL;
908	INIT_LIST_HEAD(list: &skb->tcp_tsorted_anchor);
909	return skb;
910	}
911	__kfree_skb(skb);
912	} else {
913	sk->sk_prot->enter_memory_pressure(sk);
914	sk_stream_moderate_sndbuf(sk);
915	}
916	return NULL;
917	}
918
919	static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
920	int large_allowed)
921	{
922	struct tcp_sock *tp = tcp_sk(sk);
923	u32 new_size_goal, size_goal;
924
925	if (!large_allowed)
926	return mss_now;
927
928	/* Note : tcp_tso_autosize() will eventually split this later */
929	new_size_goal = tcp_bound_to_half_wnd(tp, pktsize: sk->sk_gso_max_size);
930
931	/* We try hard to avoid divides here */
932	size_goal = tp->gso_segs * mss_now;
933	if (unlikely(new_size_goal < size_goal ||
934	new_size_goal >= size_goal + mss_now)) {
935	tp->gso_segs = min_t(u16, new_size_goal / mss_now,
936	sk->sk_gso_max_segs);
937	size_goal = tp->gso_segs * mss_now;
938	}
939
940	return max(size_goal, mss_now);
941	}
942
943	int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
944	{
945	int mss_now;
946
947	mss_now = tcp_current_mss(sk);
948	*size_goal = tcp_xmit_size_goal(sk, mss_now, large_allowed: !(flags & MSG_OOB));
949
950	return mss_now;
951	}
952
953	/* In some cases, sendmsg() could have added an skb to the write queue,
954	* but failed adding payload on it. We need to remove it to consume less
955	* memory, but more importantly be able to generate EPOLLOUT for Edge Trigger
956	* epoll() users. Another reason is that tcp_write_xmit() does not like
957	* finding an empty skb in the write queue.
958	*/
959	void tcp_remove_empty_skb(struct sock *sk)
960	{
961	struct sk_buff *skb = tcp_write_queue_tail(sk);
962
963	if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
964	tcp_unlink_write_queue(skb, sk);
965	if (tcp_write_queue_empty(sk))
966	tcp_chrono_stop(sk, type: TCP_CHRONO_BUSY);
967	tcp_wmem_free_skb(sk, skb);
968	}
969	}
970
971	/* skb changing from pure zc to mixed, must charge zc */
972	static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb)
973	{
974	if (unlikely(skb_zcopy_pure(skb))) {
975	u32 extra = skb->truesize -
976	SKB_TRUESIZE(skb_end_offset(skb));
977
978	if (!sk_wmem_schedule(sk, size: extra))
979	return -ENOMEM;
980
981	sk_mem_charge(sk, size: extra);
982	skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY;
983	}
984	return 0;
985	}
986
987
988	int tcp_wmem_schedule(struct sock *sk, int copy)
989	{
990	int left;
991
992	if (likely(sk_wmem_schedule(sk, copy)))
993	return copy;
994
995	/* We could be in trouble if we have nothing queued.
996	* Use whatever is left in sk->sk_forward_alloc and tcp_wmem[0]
997	* to guarantee some progress.
998	*/
999	left = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[0]) - sk->sk_wmem_queued;
1000	if (left > 0)
1001	sk_forced_mem_schedule(sk, min(left, copy));
1002	return min(copy, sk->sk_forward_alloc);
1003	}
1004
1005	void tcp_free_fastopen_req(struct tcp_sock *tp)
1006	{
1007	if (tp->fastopen_req) {
1008	kfree(objp: tp->fastopen_req);
1009	tp->fastopen_req = NULL;
1010	}
1011	}
1012
1013	int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
1014	size_t size, struct ubuf_info *uarg)
1015	{
1016	struct tcp_sock *tp = tcp_sk(sk);
1017	struct inet_sock *inet = inet_sk(sk);
1018	struct sockaddr *uaddr = msg->msg_name;
1019	int err, flags;
1020
1021	if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) &
1022	TFO_CLIENT_ENABLE) ||
1023	(uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1024	uaddr->sa_family == AF_UNSPEC))
1025	return -EOPNOTSUPP;
1026	if (tp->fastopen_req)
1027	return -EALREADY; /* Another Fast Open is in progress */
1028
1029	tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1030	sk->sk_allocation);
1031	if (unlikely(!tp->fastopen_req))
1032	return -ENOBUFS;
1033	tp->fastopen_req->data = msg;
1034	tp->fastopen_req->size = size;
1035	tp->fastopen_req->uarg = uarg;
1036
1037	if (inet_test_bit(DEFER_CONNECT, sk)) {
1038	err = tcp_connect(sk);
1039	/* Same failure procedure as in tcp_v4/6_connect */
1040	if (err) {
1041	tcp_set_state(sk, state: TCP_CLOSE);
1042	inet->inet_dport = 0;
1043	sk->sk_route_caps = 0;
1044	}
1045	}
1046	flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1047	err = __inet_stream_connect(sock: sk->sk_socket, uaddr,
1048	addr_len: msg->msg_namelen, flags, is_sendmsg: 1);
1049	/* fastopen_req could already be freed in __inet_stream_connect
1050	* if the connection times out or gets rst
1051	*/
1052	if (tp->fastopen_req) {
1053	*copied = tp->fastopen_req->copied;
1054	tcp_free_fastopen_req(tp);
1055	inet_clear_bit(DEFER_CONNECT, sk);
1056	}
1057	return err;
1058	}
1059
1060	int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
1061	{
1062	struct net_devmem_dmabuf_binding *binding = NULL;
1063	struct tcp_sock *tp = tcp_sk(sk);
1064	struct ubuf_info *uarg = NULL;
1065	struct sk_buff *skb;
1066	struct sockcm_cookie sockc;
1067	int flags, err, copied = 0;
1068	int mss_now = 0, size_goal, copied_syn = 0;
1069	int process_backlog = 0;
1070	int sockc_err = 0;
1071	int zc = 0;
1072	long timeo;
1073
1074	flags = msg->msg_flags;
1075
1076	sockc = (struct sockcm_cookie){ .tsflags = READ_ONCE(sk->sk_tsflags) };
1077	if (msg->msg_controllen) {
1078	sockc_err = sock_cmsg_send(sk, msg, sockc: &sockc);
1079	/* Don't return error until MSG_FASTOPEN has been processed;
1080	* that may succeed even if the cmsg is invalid.
1081	*/
1082	}
1083
1084	if ((flags & MSG_ZEROCOPY) && size) {
1085	if (msg->msg_ubuf) {
1086	uarg = msg->msg_ubuf;
1087	if (sk->sk_route_caps & NETIF_F_SG)
1088	zc = MSG_ZEROCOPY;
1089	} else if (sock_flag(sk, flag: SOCK_ZEROCOPY)) {
1090	skb = tcp_write_queue_tail(sk);
1091	uarg = msg_zerocopy_realloc(sk, size, uarg: skb_zcopy(skb),
1092	devmem: !sockc_err && sockc.dmabuf_id);
1093	if (!uarg) {
1094	err = -ENOBUFS;
1095	goto out_err;
1096	}
1097	if (sk->sk_route_caps & NETIF_F_SG)
1098	zc = MSG_ZEROCOPY;
1099	else
1100	uarg_to_msgzc(uarg)->zerocopy = 0;
1101
1102	if (!sockc_err && sockc.dmabuf_id) {
1103	binding = net_devmem_get_binding(sk, dmabuf_id: sockc.dmabuf_id);
1104	if (IS_ERR(ptr: binding)) {
1105	err = PTR_ERR(ptr: binding);
1106	binding = NULL;
1107	goto out_err;
1108	}
1109	}
1110	}
1111	} else if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES) && size) {
1112	if (sk->sk_route_caps & NETIF_F_SG)
1113	zc = MSG_SPLICE_PAGES;
1114	}
1115
1116	if (!sockc_err && sockc.dmabuf_id &&
1117	(!(flags & MSG_ZEROCOPY) || !sock_flag(sk, flag: SOCK_ZEROCOPY))) {
1118	err = -EINVAL;
1119	goto out_err;
1120	}
1121
1122	if (unlikely(flags & MSG_FASTOPEN ||
1123	inet_test_bit(DEFER_CONNECT, sk)) &&
1124	!tp->repair) {
1125	err = tcp_sendmsg_fastopen(sk, msg, copied: &copied_syn, size, uarg);
1126	if (err == -EINPROGRESS && copied_syn > 0)
1127	goto out;
1128	else if (err)
1129	goto out_err;
1130	}
1131
1132	timeo = sock_sndtimeo(sk, noblock: flags & MSG_DONTWAIT);
1133
1134	tcp_rate_check_app_limited(sk); /* is sending application-limited? */
1135
1136	/* Wait for a connection to finish. One exception is TCP Fast Open
1137	* (passive side) where data is allowed to be sent before a connection
1138	* is fully established.
1139	*/
1140	if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1141	!tcp_passive_fastopen(sk)) {
1142	err = sk_stream_wait_connect(sk, timeo_p: &timeo);
1143	if (err != 0)
1144	goto do_error;
1145	}
1146
1147	if (unlikely(tp->repair)) {
1148	if (tp->repair_queue == TCP_RECV_QUEUE) {
1149	copied = tcp_send_rcvq(sk, msg, size);
1150	goto out_nopush;
1151	}
1152
1153	err = -EINVAL;
1154	if (tp->repair_queue == TCP_NO_QUEUE)
1155	goto out_err;
1156
1157	/* 'common' sending to sendq */
1158	}
1159
1160	if (sockc_err) {
1161	err = sockc_err;
1162	goto out_err;
1163	}
1164
1165	/* This should be in poll */
1166	sk_clear_bit(nr: SOCKWQ_ASYNC_NOSPACE, sk);
1167
1168	/* Ok commence sending. */
1169	copied = 0;
1170
1171	restart:
1172	mss_now = tcp_send_mss(sk, size_goal: &size_goal, flags);
1173
1174	err = -EPIPE;
1175	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1176	goto do_error;
1177
1178	while (msg_data_left(msg)) {
1179	ssize_t copy = 0;
1180
1181	skb = tcp_write_queue_tail(sk);
1182	if (skb)
1183	copy = size_goal - skb->len;
1184
1185	trace_tcp_sendmsg_locked(sk, msg, skb, size_goal);
1186
1187	if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1188	bool first_skb;
1189
1190	new_segment:
1191	if (!sk_stream_memory_free(sk))
1192	goto wait_for_space;
1193
1194	if (unlikely(process_backlog >= 16)) {
1195	process_backlog = 0;
1196	if (sk_flush_backlog(sk))
1197	goto restart;
1198	}
1199	first_skb = tcp_rtx_and_write_queues_empty(sk);
1200	skb = tcp_stream_alloc_skb(sk, gfp: sk->sk_allocation,
1201	force_schedule: first_skb);
1202	if (!skb)
1203	goto wait_for_space;
1204
1205	process_backlog++;
1206
1207	#ifdef CONFIG_SKB_DECRYPTED
1208	skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED);
1209	#endif
1210	tcp_skb_entail(sk, skb);
1211	copy = size_goal;
1212
1213	/* All packets are restored as if they have
1214	* already been sent. skb_mstamp_ns isn't set to
1215	* avoid wrong rtt estimation.
1216	*/
1217	if (tp->repair)
1218	TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1219	}
1220
1221	/* Try to append data to the end of skb. */
1222	if (copy > msg_data_left(msg))
1223	copy = msg_data_left(msg);
1224
1225	if (zc == 0) {
1226	bool merge = true;
1227	int i = skb_shinfo(skb)->nr_frags;
1228	struct page_frag *pfrag = sk_page_frag(sk);
1229
1230	if (!sk_page_frag_refill(sk, pfrag))
1231	goto wait_for_space;
1232
1233	if (!skb_can_coalesce(skb, i, page: pfrag->page,
1234	off: pfrag->offset)) {
1235	if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
1236	tcp_mark_push(tp, skb);
1237	goto new_segment;
1238	}
1239	merge = false;
1240	}
1241
1242	copy = min_t(int, copy, pfrag->size - pfrag->offset);
1243
1244	if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) {
1245	if (tcp_downgrade_zcopy_pure(sk, skb))
1246	goto wait_for_space;
1247	skb_zcopy_downgrade_managed(skb);
1248	}
1249
1250	copy = tcp_wmem_schedule(sk, copy);
1251	if (!copy)
1252	goto wait_for_space;
1253
1254	err = skb_copy_to_page_nocache(sk, from: &msg->msg_iter, skb,
1255	page: pfrag->page,
1256	off: pfrag->offset,
1257	copy);
1258	if (err)
1259	goto do_error;
1260
1261	/* Update the skb. */
1262	if (merge) {
1263	skb_frag_size_add(frag: &skb_shinfo(skb)->frags[i - 1], delta: copy);
1264	} else {
1265	skb_fill_page_desc(skb, i, page: pfrag->page,
1266	off: pfrag->offset, size: copy);
1267	page_ref_inc(page: pfrag->page);
1268	}
1269	pfrag->offset += copy;
1270	} else if (zc == MSG_ZEROCOPY) {
1271	/* First append to a fragless skb builds initial
1272	* pure zerocopy skb
1273	*/
1274	if (!skb->len)
1275	skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY;
1276
1277	if (!skb_zcopy_pure(skb)) {
1278	copy = tcp_wmem_schedule(sk, copy);
1279	if (!copy)
1280	goto wait_for_space;
1281	}
1282
1283	err = skb_zerocopy_iter_stream(sk, skb, msg, len: copy, uarg,
1284	binding);
1285	if (err == -EMSGSIZE || err == -EEXIST) {
1286	tcp_mark_push(tp, skb);
1287	goto new_segment;
1288	}
1289	if (err < 0)
1290	goto do_error;
1291	copy = err;
1292	} else if (zc == MSG_SPLICE_PAGES) {
1293	/* Splice in data if we can; copy if we can't. */
1294	if (tcp_downgrade_zcopy_pure(sk, skb))
1295	goto wait_for_space;
1296	copy = tcp_wmem_schedule(sk, copy);
1297	if (!copy)
1298	goto wait_for_space;
1299
1300	err = skb_splice_from_iter(skb, iter: &msg->msg_iter, maxsize: copy,
1301	gfp: sk->sk_allocation);
1302	if (err < 0) {
1303	if (err == -EMSGSIZE) {
1304	tcp_mark_push(tp, skb);
1305	goto new_segment;
1306	}
1307	goto do_error;
1308	}
1309	copy = err;
1310
1311	if (!(flags & MSG_NO_SHARED_FRAGS))
1312	skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
1313
1314	sk_wmem_queued_add(sk, val: copy);
1315	sk_mem_charge(sk, size: copy);
1316	}
1317
1318	if (!copied)
1319	TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1320
1321	WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1322	TCP_SKB_CB(skb)->end_seq += copy;
1323	tcp_skb_pcount_set(skb, segs: 0);
1324
1325	copied += copy;
1326	if (!msg_data_left(msg)) {
1327	if (unlikely(flags & MSG_EOR))
1328	TCP_SKB_CB(skb)->eor = 1;
1329	goto out;
1330	}
1331
1332	if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair))
1333	continue;
1334
1335	if (forced_push(tp)) {
1336	tcp_mark_push(tp, skb);
1337	__tcp_push_pending_frames(sk, cur_mss: mss_now, TCP_NAGLE_PUSH);
1338	} else if (skb == tcp_send_head(sk))
1339	tcp_push_one(sk, mss_now);
1340	continue;
1341
1342	wait_for_space:
1343	set_bit(nr: SOCK_NOSPACE, addr: &sk->sk_socket->flags);
1344	tcp_remove_empty_skb(sk);
1345	if (copied)
1346	tcp_push(sk, flags: flags & ~MSG_MORE, mss_now,
1347	TCP_NAGLE_PUSH, size_goal);
1348
1349	err = sk_stream_wait_memory(sk, timeo_p: &timeo);
1350	if (err != 0)
1351	goto do_error;
1352
1353	mss_now = tcp_send_mss(sk, size_goal: &size_goal, flags);
1354	}
1355
1356	out:
1357	if (copied) {
1358	tcp_tx_timestamp(sk, sockc: &sockc);
1359	tcp_push(sk, flags, mss_now, nonagle: tp->nonagle, size_goal);
1360	}
1361	out_nopush:
1362	/* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1363	if (uarg && !msg->msg_ubuf)
1364	net_zcopy_put(uarg);
1365	if (binding)
1366	net_devmem_dmabuf_binding_put(binding);
1367	return copied + copied_syn;
1368
1369	do_error:
1370	tcp_remove_empty_skb(sk);
1371
1372	if (copied + copied_syn)
1373	goto out;
1374	out_err:
1375	/* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1376	if (uarg && !msg->msg_ubuf)
1377	net_zcopy_put_abort(uarg, have_uref: true);
1378	err = sk_stream_error(sk, flags, err);
1379	/* make sure we wake any epoll edge trigger waiter */
1380	if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1381	sk->sk_write_space(sk);
1382	tcp_chrono_stop(sk, type: TCP_CHRONO_SNDBUF_LIMITED);
1383	}
1384	if (binding)
1385	net_devmem_dmabuf_binding_put(binding);
1386
1387	return err;
1388	}
1389	EXPORT_SYMBOL_GPL(tcp_sendmsg_locked);
1390
1391	int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1392	{
1393	int ret;
1394
1395	lock_sock(sk);
1396	ret = tcp_sendmsg_locked(sk, msg, size);
1397	release_sock(sk);
1398
1399	return ret;
1400	}
1401	EXPORT_SYMBOL(tcp_sendmsg);
1402
1403	void tcp_splice_eof(struct socket *sock)
1404	{
1405	struct sock *sk = sock->sk;
1406	struct tcp_sock *tp = tcp_sk(sk);
1407	int mss_now, size_goal;
1408
1409	if (!tcp_write_queue_tail(sk))
1410	return;
1411
1412	lock_sock(sk);
1413	mss_now = tcp_send_mss(sk, size_goal: &size_goal, flags: 0);
1414	tcp_push(sk, flags: 0, mss_now, nonagle: tp->nonagle, size_goal);
1415	release_sock(sk);
1416	}
1417	EXPORT_IPV6_MOD_GPL(tcp_splice_eof);
1418
1419	/*
1420	* Handle reading urgent data. BSD has very simple semantics for
1421	* this, no blocking and very strange errors 8)
1422	*/
1423
1424	static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1425	{
1426	struct tcp_sock *tp = tcp_sk(sk);
1427
1428	/* No URG data to read. */
1429	if (sock_flag(sk, flag: SOCK_URGINLINE) || !tp->urg_data ||
1430	tp->urg_data == TCP_URG_READ)
1431	return -EINVAL; /* Yes this is right ! */
1432
1433	if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, flag: SOCK_DONE))
1434	return -ENOTCONN;
1435
1436	if (tp->urg_data & TCP_URG_VALID) {
1437	int err = 0;
1438	char c = tp->urg_data;
1439
1440	if (!(flags & MSG_PEEK))
1441	WRITE_ONCE(tp->urg_data, TCP_URG_READ);
1442
1443	/* Read urgent data. */
1444	msg->msg_flags |= MSG_OOB;
1445
1446	if (len > 0) {
1447	if (!(flags & MSG_TRUNC))
1448	err = memcpy_to_msg(msg, data: &c, len: 1);
1449	len = 1;
1450	} else
1451	msg->msg_flags |= MSG_TRUNC;
1452
1453	return err ? -EFAULT : len;
1454	}
1455
1456	if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1457	return 0;
1458
1459	/* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and
1460	* the available implementations agree in this case:
1461	* this call should never block, independent of the
1462	* blocking state of the socket.
1463	* Mike <pall@rz.uni-karlsruhe.de>
1464	*/
1465	return -EAGAIN;
1466	}
1467
1468	static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1469	{
1470	struct sk_buff *skb;
1471	int copied = 0, err = 0;
1472
1473	skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
1474	err = skb_copy_datagram_msg(from: skb, offset: 0, msg, size: skb->len);
1475	if (err)
1476	return err;
1477	copied += skb->len;
1478	}
1479
1480	skb_queue_walk(&sk->sk_write_queue, skb) {
1481	err = skb_copy_datagram_msg(from: skb, offset: 0, msg, size: skb->len);
1482	if (err)
1483	break;
1484
1485	copied += skb->len;
1486	}
1487
1488	return err ?: copied;
1489	}
1490
1491	/* Clean up the receive buffer for full frames taken by the user,
1492	* then send an ACK if necessary. COPIED is the number of bytes
1493	* tcp_recvmsg has given to the user so far, it speeds up the
1494	* calculation of whether or not we must ACK for the sake of
1495	* a window update.
1496	*/
1497	void __tcp_cleanup_rbuf(struct sock *sk, int copied)
1498	{
1499	struct tcp_sock *tp = tcp_sk(sk);
1500	bool time_to_ack = false;
1501
1502	if (inet_csk_ack_scheduled(sk)) {
1503	const struct inet_connection_sock *icsk = inet_csk(sk);
1504
1505	if (/* Once-per-two-segments ACK was not sent by tcp_input.c */
1506	tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1507	/*
1508	* If this read emptied read buffer, we send ACK, if
1509	* connection is not bidirectional, user drained
1510	* receive buffer and there was a small segment
1511	* in queue.
1512	*/
1513	(copied > 0 &&
1514	((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1515	((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1516	!inet_csk_in_pingpong_mode(sk))) &&
1517	!atomic_read(v: &sk->sk_rmem_alloc)))
1518	time_to_ack = true;
1519	}
1520
1521	/* We send an ACK if we can now advertise a non-zero window
1522	* which has been raised "significantly".
1523	*
1524	* Even if window raised up to infinity, do not send window open ACK
1525	* in states, where we will not receive more. It is useless.
1526	*/
1527	if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1528	__u32 rcv_window_now = tcp_receive_window(tp);
1529
1530	/* Optimize, __tcp_select_window() is not cheap. */
1531	if (2*rcv_window_now <= tp->window_clamp) {
1532	__u32 new_window = __tcp_select_window(sk);
1533
1534	/* Send ACK now, if this read freed lots of space
1535	* in our buffer. Certainly, new_window is new window.
1536	* We can advertise it now, if it is not less than current one.
1537	* "Lots" means "at least twice" here.
1538	*/
1539	if (new_window && new_window >= 2 * rcv_window_now)
1540	time_to_ack = true;
1541	}
1542	}
1543	if (time_to_ack)
1544	tcp_send_ack(sk);
1545	}
1546
1547	void tcp_cleanup_rbuf(struct sock *sk, int copied)
1548	{
1549	struct sk_buff *skb = skb_peek(list_: &sk->sk_receive_queue);
1550	struct tcp_sock *tp = tcp_sk(sk);
1551
1552	WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1553	"cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1554	tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1555	__tcp_cleanup_rbuf(sk, copied);
1556	}
1557
1558	static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb)
1559	{
1560	__skb_unlink(skb, list: &sk->sk_receive_queue);
1561	if (likely(skb->destructor == sock_rfree)) {
1562	sock_rfree(skb);
1563	skb->destructor = NULL;
1564	skb->sk = NULL;
1565	return skb_attempt_defer_free(skb);
1566	}
1567	__kfree_skb(skb);
1568	}
1569
1570	struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1571	{
1572	struct sk_buff *skb;
1573	u32 offset;
1574
1575	while ((skb = skb_peek(list_: &sk->sk_receive_queue)) != NULL) {
1576	offset = seq - TCP_SKB_CB(skb)->seq;
1577	if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1578	pr_err_once("%s: found a SYN, please report !\n", __func__);
1579	offset--;
1580	}
1581	if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1582	*off = offset;
1583	return skb;
1584	}
1585	/* This looks weird, but this can happen if TCP collapsing
1586	* splitted a fat GRO packet, while we released socket lock
1587	* in skb_splice_bits()
1588	*/
1589	tcp_eat_recv_skb(sk, skb);
1590	}
1591	return NULL;
1592	}
1593	EXPORT_SYMBOL(tcp_recv_skb);
1594
1595	/*
1596	* This routine provides an alternative to tcp_recvmsg() for routines
1597	* that would like to handle copying from skbuffs directly in 'sendfile'
1598	* fashion.
1599	* Note:
1600	* - It is assumed that the socket was locked by the caller.
1601	* - The routine does not block.
1602	* - At present, there is no support for reading OOB data
1603	* or for 'peeking' the socket using this routine
1604	* (although both would be easy to implement).
1605	*/
1606	static int __tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1607	sk_read_actor_t recv_actor, bool noack,
1608	u32 *copied_seq)
1609	{
1610	struct sk_buff *skb;
1611	struct tcp_sock *tp = tcp_sk(sk);
1612	u32 seq = *copied_seq;
1613	u32 offset;
1614	int copied = 0;
1615
1616	if (sk->sk_state == TCP_LISTEN)
1617	return -ENOTCONN;
1618	while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1619	if (offset < skb->len) {
1620	int used;
1621	size_t len;
1622
1623	len = skb->len - offset;
1624	/* Stop reading if we hit a patch of urgent data */
1625	if (unlikely(tp->urg_data)) {
1626	u32 urg_offset = tp->urg_seq - seq;
1627	if (urg_offset < len)
1628	len = urg_offset;
1629	if (!len)
1630	break;
1631	}
1632	used = recv_actor(desc, skb, offset, len);
1633	if (used <= 0) {
1634	if (!copied)
1635	copied = used;
1636	break;
1637	}
1638	if (WARN_ON_ONCE(used > len))
1639	used = len;
1640	seq += used;
1641	copied += used;
1642	offset += used;
1643
1644	/* If recv_actor drops the lock (e.g. TCP splice
1645	* receive) the skb pointer might be invalid when
1646	* getting here: tcp_collapse might have deleted it
1647	* while aggregating skbs from the socket queue.
1648	*/
1649	skb = tcp_recv_skb(sk, seq - 1, &offset);
1650	if (!skb)
1651	break;
1652	/* TCP coalescing might have appended data to the skb.
1653	* Try to splice more frags
1654	*/
1655	if (offset + 1 != skb->len)
1656	continue;
1657	}
1658	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1659	tcp_eat_recv_skb(sk, skb);
1660	++seq;
1661	break;
1662	}
1663	tcp_eat_recv_skb(sk, skb);
1664	if (!desc->count)
1665	break;
1666	WRITE_ONCE(*copied_seq, seq);
1667	}
1668	WRITE_ONCE(*copied_seq, seq);
1669
1670	if (noack)
1671	goto out;
1672
1673	tcp_rcv_space_adjust(sk);
1674
1675	/* Clean up data we have read: This will do ACK frames. */
1676	if (copied > 0) {
1677	tcp_recv_skb(sk, seq, &offset);
1678	tcp_cleanup_rbuf(sk, copied);
1679	}
1680	out:
1681	return copied;
1682	}
1683
1684	int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1685	sk_read_actor_t recv_actor)
1686	{
1687	return __tcp_read_sock(sk, desc, recv_actor, noack: false,
1688	copied_seq: &tcp_sk(sk)->copied_seq);
1689	}
1690	EXPORT_SYMBOL(tcp_read_sock);
1691
1692	int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc,
1693	sk_read_actor_t recv_actor, bool noack,
1694	u32 *copied_seq)
1695	{
1696	return __tcp_read_sock(sk, desc, recv_actor, noack, copied_seq);
1697	}
1698
1699	int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
1700	{
1701	struct sk_buff *skb;
1702	int copied = 0;
1703
1704	if (sk->sk_state == TCP_LISTEN)
1705	return -ENOTCONN;
1706
1707	while ((skb = skb_peek(list_: &sk->sk_receive_queue)) != NULL) {
1708	u8 tcp_flags;
1709	int used;
1710
1711	__skb_unlink(skb, list: &sk->sk_receive_queue);
1712	WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
1713	tcp_flags = TCP_SKB_CB(skb)->tcp_flags;
1714	used = recv_actor(sk, skb);
1715	if (used < 0) {
1716	if (!copied)
1717	copied = used;
1718	break;
1719	}
1720	copied += used;
1721
1722	if (tcp_flags & TCPHDR_FIN)
1723	break;
1724	}
1725	return copied;
1726	}
1727	EXPORT_IPV6_MOD(tcp_read_skb);
1728
1729	void tcp_read_done(struct sock *sk, size_t len)
1730	{
1731	struct tcp_sock *tp = tcp_sk(sk);
1732	u32 seq = tp->copied_seq;
1733	struct sk_buff *skb;
1734	size_t left;
1735	u32 offset;
1736
1737	if (sk->sk_state == TCP_LISTEN)
1738	return;
1739
1740	left = len;
1741	while (left && (skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1742	int used;
1743
1744	used = min_t(size_t, skb->len - offset, left);
1745	seq += used;
1746	left -= used;
1747
1748	if (skb->len > offset + used)
1749	break;
1750
1751	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1752	tcp_eat_recv_skb(sk, skb);
1753	++seq;
1754	break;
1755	}
1756	tcp_eat_recv_skb(sk, skb);
1757	}
1758	WRITE_ONCE(tp->copied_seq, seq);
1759
1760	tcp_rcv_space_adjust(sk);
1761
1762	/* Clean up data we have read: This will do ACK frames. */
1763	if (left != len)
1764	tcp_cleanup_rbuf(sk, copied: len - left);
1765	}
1766	EXPORT_SYMBOL(tcp_read_done);
1767
1768	int tcp_peek_len(struct socket *sock)
1769	{
1770	return tcp_inq(sk: sock->sk);
1771	}
1772	EXPORT_IPV6_MOD(tcp_peek_len);
1773
1774	/* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */
1775	int tcp_set_rcvlowat(struct sock *sk, int val)
1776	{
1777	int space, cap;
1778
1779	if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1780	cap = sk->sk_rcvbuf >> 1;
1781	else
1782	cap = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1;
1783	val = min(val, cap);
1784	WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1785
1786	/* Check if we need to signal EPOLLIN right now */
1787	tcp_data_ready(sk);
1788
1789	if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1790	return 0;
1791
1792	space = tcp_space_from_win(sk, win: val);
1793	if (space > sk->sk_rcvbuf) {
1794	WRITE_ONCE(sk->sk_rcvbuf, space);
1795	WRITE_ONCE(tcp_sk(sk)->window_clamp, val);
1796	}
1797	return 0;
1798	}
1799	EXPORT_IPV6_MOD(tcp_set_rcvlowat);
1800
1801	void tcp_update_recv_tstamps(struct sk_buff *skb,
1802	struct scm_timestamping_internal *tss)
1803	{
1804	if (skb->tstamp)
1805	tss->ts[0] = ktime_to_timespec64(skb->tstamp);
1806	else
1807	tss->ts[0] = (struct timespec64) {0};
1808
1809	if (skb_hwtstamps(skb)->hwtstamp)
1810	tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp);
1811	else
1812	tss->ts[2] = (struct timespec64) {0};
1813	}
1814
1815	#ifdef CONFIG_MMU
1816	static const struct vm_operations_struct tcp_vm_ops = {
1817	};
1818
1819	int tcp_mmap(struct file *file, struct socket *sock,
1820	struct vm_area_struct *vma)
1821	{
1822	if (vma->vm_flags & (VM_WRITE | VM_EXEC))
1823	return -EPERM;
1824	vm_flags_clear(vma, VM_MAYWRITE | VM_MAYEXEC);
1825
1826	/* Instruct vm_insert_page() to not mmap_read_lock(mm) */
1827	vm_flags_set(vma, VM_MIXEDMAP);
1828
1829	vma->vm_ops = &tcp_vm_ops;
1830	return 0;
1831	}
1832	EXPORT_IPV6_MOD(tcp_mmap);
1833
1834	static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb,
1835	u32 *offset_frag)
1836	{
1837	skb_frag_t *frag;
1838
1839	if (unlikely(offset_skb >= skb->len))
1840	return NULL;
1841
1842	offset_skb -= skb_headlen(skb);
1843	if ((int)offset_skb < 0 || skb_has_frag_list(skb))
1844	return NULL;
1845
1846	frag = skb_shinfo(skb)->frags;
1847	while (offset_skb) {
1848	if (skb_frag_size(frag) > offset_skb) {
1849	*offset_frag = offset_skb;
1850	return frag;
1851	}
1852	offset_skb -= skb_frag_size(frag);
1853	++frag;
1854	}
1855	*offset_frag = 0;
1856	return frag;
1857	}
1858
1859	static bool can_map_frag(const skb_frag_t *frag)
1860	{
1861	struct page *page;
1862
1863	if (skb_frag_size(frag) != PAGE_SIZE || skb_frag_off(frag))
1864	return false;
1865
1866	page = skb_frag_page(frag);
1867
1868	if (PageCompound(page) || page->mapping)
1869	return false;
1870
1871	return true;
1872	}
1873
1874	static int find_next_mappable_frag(const skb_frag_t *frag,
1875	int remaining_in_skb)
1876	{
1877	int offset = 0;
1878
1879	if (likely(can_map_frag(frag)))
1880	return 0;
1881
1882	while (offset < remaining_in_skb && !can_map_frag(frag)) {
1883	offset += skb_frag_size(frag);
1884	++frag;
1885	}
1886	return offset;
1887	}
1888
1889	static void tcp_zerocopy_set_hint_for_skb(struct sock *sk,
1890	struct tcp_zerocopy_receive *zc,
1891	struct sk_buff *skb, u32 offset)
1892	{
1893	u32 frag_offset, partial_frag_remainder = 0;
1894	int mappable_offset;
1895	skb_frag_t *frag;
1896
1897	/* worst case: skip to next skb. try to improve on this case below */
1898	zc->recv_skip_hint = skb->len - offset;
1899
1900	/* Find the frag containing this offset (and how far into that frag) */
1901	frag = skb_advance_to_frag(skb, offset_skb: offset, offset_frag: &frag_offset);
1902	if (!frag)
1903	return;
1904
1905	if (frag_offset) {
1906	struct skb_shared_info *info = skb_shinfo(skb);
1907
1908	/* We read part of the last frag, must recvmsg() rest of skb. */
1909	if (frag == &info->frags[info->nr_frags - 1])
1910	return;
1911
1912	/* Else, we must at least read the remainder in this frag. */
1913	partial_frag_remainder = skb_frag_size(frag) - frag_offset;
1914	zc->recv_skip_hint -= partial_frag_remainder;
1915	++frag;
1916	}
1917
1918	/* partial_frag_remainder: If part way through a frag, must read rest.
1919	* mappable_offset: Bytes till next mappable frag, *not* counting bytes
1920	* in partial_frag_remainder.
1921	*/
1922	mappable_offset = find_next_mappable_frag(frag, remaining_in_skb: zc->recv_skip_hint);
1923	zc->recv_skip_hint = mappable_offset + partial_frag_remainder;
1924	}
1925
1926	static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
1927	int flags, struct scm_timestamping_internal *tss,
1928	int *cmsg_flags);
1929	static int receive_fallback_to_copy(struct sock *sk,
1930	struct tcp_zerocopy_receive *zc, int inq,
1931	struct scm_timestamping_internal *tss)
1932	{
1933	unsigned long copy_address = (unsigned long)zc->copybuf_address;
1934	struct msghdr msg = {};
1935	int err;
1936
1937	zc->length = 0;
1938	zc->recv_skip_hint = 0;
1939
1940	if (copy_address != zc->copybuf_address)
1941	return -EINVAL;
1942
1943	err = import_ubuf(ITER_DEST, buf: (void __user *)copy_address, len: inq,
1944	i: &msg.msg_iter);
1945	if (err)
1946	return err;
1947
1948	err = tcp_recvmsg_locked(sk, msg: &msg, len: inq, MSG_DONTWAIT,
1949	tss, cmsg_flags: &zc->msg_flags);
1950	if (err < 0)
1951	return err;
1952
1953	zc->copybuf_len = err;
1954	if (likely(zc->copybuf_len)) {
1955	struct sk_buff *skb;
1956	u32 offset;
1957
1958	skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset);
1959	if (skb)
1960	tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset);
1961	}
1962	return 0;
1963	}
1964
1965	static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc,
1966	struct sk_buff *skb, u32 copylen,
1967	u32 *offset, u32 *seq)
1968	{
1969	unsigned long copy_address = (unsigned long)zc->copybuf_address;
1970	struct msghdr msg = {};
1971	int err;
1972
1973	if (copy_address != zc->copybuf_address)
1974	return -EINVAL;
1975
1976	err = import_ubuf(ITER_DEST, buf: (void __user *)copy_address, len: copylen,
1977	i: &msg.msg_iter);
1978	if (err)
1979	return err;
1980	err = skb_copy_datagram_msg(from: skb, offset: *offset, msg: &msg, size: copylen);
1981	if (err)
1982	return err;
1983	zc->recv_skip_hint -= copylen;
1984	*offset += copylen;
1985	*seq += copylen;
1986	return (__s32)copylen;
1987	}
1988
1989	static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc,
1990	struct sock *sk,
1991	struct sk_buff *skb,
1992	u32 *seq,
1993	s32 copybuf_len,
1994	struct scm_timestamping_internal *tss)
1995	{
1996	u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint);
1997
1998	if (!copylen)
1999	return 0;
2000	/* skb is null if inq < PAGE_SIZE. */
2001	if (skb) {
2002	offset = *seq - TCP_SKB_CB(skb)->seq;
2003	} else {
2004	skb = tcp_recv_skb(sk, *seq, &offset);
2005	if (TCP_SKB_CB(skb)->has_rxtstamp) {
2006	tcp_update_recv_tstamps(skb, tss);
2007	zc->msg_flags |= TCP_CMSG_TS;
2008	}
2009	}
2010
2011	zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, offset: &offset,
2012	seq);
2013	return zc->copybuf_len < 0 ? 0 : copylen;
2014	}
2015
2016	static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma,
2017	struct page **pending_pages,
2018	unsigned long pages_remaining,
2019	unsigned long *address,
2020	u32 *length,
2021	u32 *seq,
2022	struct tcp_zerocopy_receive *zc,
2023	u32 total_bytes_to_map,
2024	int err)
2025	{
2026	/* At least one page did not map. Try zapping if we skipped earlier. */
2027	if (err == -EBUSY &&
2028	zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) {
2029	u32 maybe_zap_len;
2030
2031	maybe_zap_len = total_bytes_to_map - /* All bytes to map */
2032	*length + /* Mapped or pending */
2033	(pages_remaining * PAGE_SIZE); /* Failed map. */
2034	zap_page_range_single(vma, address: *address, size: maybe_zap_len, NULL);
2035	err = 0;
2036	}
2037
2038	if (!err) {
2039	unsigned long leftover_pages = pages_remaining;
2040	int bytes_mapped;
2041
2042	/* We called zap_page_range_single, try to reinsert. */
2043	err = vm_insert_pages(vma, addr: *address,
2044	pages: pending_pages,
2045	num: &pages_remaining);
2046	bytes_mapped = PAGE_SIZE * (leftover_pages - pages_remaining);
2047	*seq += bytes_mapped;
2048	*address += bytes_mapped;
2049	}
2050	if (err) {
2051	/* Either we were unable to zap, OR we zapped, retried an
2052	* insert, and still had an issue. Either ways, pages_remaining
2053	* is the number of pages we were unable to map, and we unroll
2054	* some state we speculatively touched before.
2055	*/
2056	const int bytes_not_mapped = PAGE_SIZE * pages_remaining;
2057
2058	*length -= bytes_not_mapped;
2059	zc->recv_skip_hint += bytes_not_mapped;
2060	}
2061	return err;
2062	}
2063
2064	static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma,
2065	struct page **pages,
2066	unsigned int pages_to_map,
2067	unsigned long *address,
2068	u32 *length,
2069	u32 *seq,
2070	struct tcp_zerocopy_receive *zc,
2071	u32 total_bytes_to_map)
2072	{
2073	unsigned long pages_remaining = pages_to_map;
2074	unsigned int pages_mapped;
2075	unsigned int bytes_mapped;
2076	int err;
2077
2078	err = vm_insert_pages(vma, addr: *address, pages, num: &pages_remaining);
2079	pages_mapped = pages_to_map - (unsigned int)pages_remaining;
2080	bytes_mapped = PAGE_SIZE * pages_mapped;
2081	/* Even if vm_insert_pages fails, it may have partially succeeded in
2082	* mapping (some but not all of the pages).
2083	*/
2084	*seq += bytes_mapped;
2085	*address += bytes_mapped;
2086
2087	if (likely(!err))
2088	return 0;
2089
2090	/* Error: maybe zap and retry + rollback state for failed inserts. */
2091	return tcp_zerocopy_vm_insert_batch_error(vma, pending_pages: pages + pages_mapped,
2092	pages_remaining, address, length, seq, zc, total_bytes_to_map,
2093	err);
2094	}
2095
2096	#define TCP_VALID_ZC_MSG_FLAGS (TCP_CMSG_TS)
2097	static void tcp_zc_finalize_rx_tstamp(struct sock *sk,
2098	struct tcp_zerocopy_receive *zc,
2099	struct scm_timestamping_internal *tss)
2100	{
2101	unsigned long msg_control_addr;
2102	struct msghdr cmsg_dummy;
2103
2104	msg_control_addr = (unsigned long)zc->msg_control;
2105	cmsg_dummy.msg_control_user = (void __user *)msg_control_addr;
2106	cmsg_dummy.msg_controllen =
2107	(__kernel_size_t)zc->msg_controllen;
2108	cmsg_dummy.msg_flags = in_compat_syscall()
2109	? MSG_CMSG_COMPAT : 0;
2110	cmsg_dummy.msg_control_is_user = true;
2111	zc->msg_flags = 0;
2112	if (zc->msg_control == msg_control_addr &&
2113	zc->msg_controllen == cmsg_dummy.msg_controllen) {
2114	tcp_recv_timestamp(msg: &cmsg_dummy, sk, tss);
2115	zc->msg_control = (__u64)
2116	((uintptr_t)cmsg_dummy.msg_control_user);
2117	zc->msg_controllen =
2118	(__u64)cmsg_dummy.msg_controllen;
2119	zc->msg_flags = (__u32)cmsg_dummy.msg_flags;
2120	}
2121	}
2122
2123	static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm,
2124	unsigned long address,
2125	bool *mmap_locked)
2126	{
2127	struct vm_area_struct *vma = lock_vma_under_rcu(mm, address);
2128
2129	if (vma) {
2130	if (vma->vm_ops != &tcp_vm_ops) {
2131	vma_end_read(vma);
2132	return NULL;
2133	}
2134	*mmap_locked = false;
2135	return vma;
2136	}
2137
2138	mmap_read_lock(mm);
2139	vma = vma_lookup(mm, addr: address);
2140	if (!vma || vma->vm_ops != &tcp_vm_ops) {
2141	mmap_read_unlock(mm);
2142	return NULL;
2143	}
2144	*mmap_locked = true;
2145	return vma;
2146	}
2147
2148	#define TCP_ZEROCOPY_PAGE_BATCH_SIZE 32
2149	static int tcp_zerocopy_receive(struct sock *sk,
2150	struct tcp_zerocopy_receive *zc,
2151	struct scm_timestamping_internal *tss)
2152	{
2153	u32 length = 0, offset, vma_len, avail_len, copylen = 0;
2154	unsigned long address = (unsigned long)zc->address;
2155	struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE];
2156	s32 copybuf_len = zc->copybuf_len;
2157	struct tcp_sock *tp = tcp_sk(sk);
2158	const skb_frag_t *frags = NULL;
2159	unsigned int pages_to_map = 0;
2160	struct vm_area_struct *vma;
2161	struct sk_buff *skb = NULL;
2162	u32 seq = tp->copied_seq;
2163	u32 total_bytes_to_map;
2164	int inq = tcp_inq(sk);
2165	bool mmap_locked;
2166	int ret;
2167
2168	zc->copybuf_len = 0;
2169	zc->msg_flags = 0;
2170
2171	if (address & (PAGE_SIZE - 1) || address != zc->address)
2172	return -EINVAL;
2173
2174	if (sk->sk_state == TCP_LISTEN)
2175	return -ENOTCONN;
2176
2177	sock_rps_record_flow(sk);
2178
2179	if (inq && inq <= copybuf_len)
2180	return receive_fallback_to_copy(sk, zc, inq, tss);
2181
2182	if (inq < PAGE_SIZE) {
2183	zc->length = 0;
2184	zc->recv_skip_hint = inq;
2185	if (!inq && sock_flag(sk, flag: SOCK_DONE))
2186	return -EIO;
2187	return 0;
2188	}
2189
2190	vma = find_tcp_vma(current->mm, address, mmap_locked: &mmap_locked);
2191	if (!vma)
2192	return -EINVAL;
2193
2194	vma_len = min_t(unsigned long, zc->length, vma->vm_end - address);
2195	avail_len = min_t(u32, vma_len, inq);
2196	total_bytes_to_map = avail_len & ~(PAGE_SIZE - 1);
2197	if (total_bytes_to_map) {
2198	if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT))
2199	zap_page_range_single(vma, address, size: total_bytes_to_map,
2200	NULL);
2201	zc->length = total_bytes_to_map;
2202	zc->recv_skip_hint = 0;
2203	} else {
2204	zc->length = avail_len;
2205	zc->recv_skip_hint = avail_len;
2206	}
2207	ret = 0;
2208	while (length + PAGE_SIZE <= zc->length) {
2209	int mappable_offset;
2210	struct page *page;
2211
2212	if (zc->recv_skip_hint < PAGE_SIZE) {
2213	u32 offset_frag;
2214
2215	if (skb) {
2216	if (zc->recv_skip_hint > 0)
2217	break;
2218	skb = skb->next;
2219	offset = seq - TCP_SKB_CB(skb)->seq;
2220	} else {
2221	skb = tcp_recv_skb(sk, seq, &offset);
2222	}
2223
2224	if (!skb_frags_readable(skb))
2225	break;
2226
2227	if (TCP_SKB_CB(skb)->has_rxtstamp) {
2228	tcp_update_recv_tstamps(skb, tss);
2229	zc->msg_flags |= TCP_CMSG_TS;
2230	}
2231	zc->recv_skip_hint = skb->len - offset;
2232	frags = skb_advance_to_frag(skb, offset_skb: offset, offset_frag: &offset_frag);
2233	if (!frags || offset_frag)
2234	break;
2235	}
2236
2237	mappable_offset = find_next_mappable_frag(frag: frags,
2238	remaining_in_skb: zc->recv_skip_hint);
2239	if (mappable_offset) {
2240	zc->recv_skip_hint = mappable_offset;
2241	break;
2242	}
2243	page = skb_frag_page(frag: frags);
2244	if (WARN_ON_ONCE(!page))
2245	break;
2246
2247	prefetchw(x: page);
2248	pages[pages_to_map++] = page;
2249	length += PAGE_SIZE;
2250	zc->recv_skip_hint -= PAGE_SIZE;
2251	frags++;
2252	if (pages_to_map == TCP_ZEROCOPY_PAGE_BATCH_SIZE ||
2253	zc->recv_skip_hint < PAGE_SIZE) {
2254	/* Either full batch, or we're about to go to next skb
2255	* (and we cannot unroll failed ops across skbs).
2256	*/
2257	ret = tcp_zerocopy_vm_insert_batch(vma, pages,
2258	pages_to_map,
2259	address: &address, length: &length,
2260	seq: &seq, zc,
2261	total_bytes_to_map);
2262	if (ret)
2263	goto out;
2264	pages_to_map = 0;
2265	}
2266	}
2267	if (pages_to_map) {
2268	ret = tcp_zerocopy_vm_insert_batch(vma, pages, pages_to_map,
2269	address: &address, length: &length, seq: &seq,
2270	zc, total_bytes_to_map);
2271	}
2272	out:
2273	if (mmap_locked)
2274	mmap_read_unlock(current->mm);
2275	else
2276	vma_end_read(vma);
2277	/* Try to copy straggler data. */
2278	if (!ret)
2279	copylen = tcp_zc_handle_leftover(zc, sk, skb, seq: &seq, copybuf_len, tss);
2280
2281	if (length + copylen) {
2282	WRITE_ONCE(tp->copied_seq, seq);
2283	tcp_rcv_space_adjust(sk);
2284
2285	/* Clean up data we have read: This will do ACK frames. */
2286	tcp_recv_skb(sk, seq, &offset);
2287	tcp_cleanup_rbuf(sk, copied: length + copylen);
2288	ret = 0;
2289	if (length == zc->length)
2290	zc->recv_skip_hint = 0;
2291	} else {
2292	if (!zc->recv_skip_hint && sock_flag(sk, flag: SOCK_DONE))
2293	ret = -EIO;
2294	}
2295	zc->length = length;
2296	return ret;
2297	}
2298	#endif
2299
2300	/* Similar to __sock_recv_timestamp, but does not require an skb */
2301	void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
2302	struct scm_timestamping_internal *tss)
2303	{
2304	int new_tstamp = sock_flag(sk, flag: SOCK_TSTAMP_NEW);
2305	u32 tsflags = READ_ONCE(sk->sk_tsflags);
2306	bool has_timestamping = false;
2307
2308	if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) {
2309	if (sock_flag(sk, flag: SOCK_RCVTSTAMP)) {
2310	if (sock_flag(sk, flag: SOCK_RCVTSTAMPNS)) {
2311	if (new_tstamp) {
2312	struct __kernel_timespec kts = {
2313	.tv_sec = tss->ts[0].tv_sec,
2314	.tv_nsec = tss->ts[0].tv_nsec,
2315	};
2316	put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
2317	len: sizeof(kts), data: &kts);
2318	} else {
2319	struct __kernel_old_timespec ts_old = {
2320	.tv_sec = tss->ts[0].tv_sec,
2321	.tv_nsec = tss->ts[0].tv_nsec,
2322	};
2323	put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
2324	len: sizeof(ts_old), data: &ts_old);
2325	}
2326	} else {
2327	if (new_tstamp) {
2328	struct __kernel_sock_timeval stv = {
2329	.tv_sec = tss->ts[0].tv_sec,
2330	.tv_usec = tss->ts[0].tv_nsec / 1000,
2331	};
2332	put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
2333	len: sizeof(stv), data: &stv);
2334	} else {
2335	struct __kernel_old_timeval tv = {
2336	.tv_sec = tss->ts[0].tv_sec,
2337	.tv_usec = tss->ts[0].tv_nsec / 1000,
2338	};
2339	put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
2340	len: sizeof(tv), data: &tv);
2341	}
2342	}
2343	}
2344
2345	if (tsflags & SOF_TIMESTAMPING_SOFTWARE &&
2346	(tsflags & SOF_TIMESTAMPING_RX_SOFTWARE ||
2347	!(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER)))
2348	has_timestamping = true;
2349	else
2350	tss->ts[0] = (struct timespec64) {0};
2351	}
2352
2353	if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) {
2354	if (tsflags & SOF_TIMESTAMPING_RAW_HARDWARE &&
2355	(tsflags & SOF_TIMESTAMPING_RX_HARDWARE ||
2356	!(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER)))
2357	has_timestamping = true;
2358	else
2359	tss->ts[2] = (struct timespec64) {0};
2360	}
2361
2362	if (has_timestamping) {
2363	tss->ts[1] = (struct timespec64) {0};
2364	if (sock_flag(sk, SOCK_TSTAMP_NEW))
2365	put_cmsg_scm_timestamping64(msg, tss);
2366	else
2367	put_cmsg_scm_timestamping(msg, tss);
2368	}
2369	}
2370
2371	static int tcp_inq_hint(struct sock *sk)
2372	{
2373	const struct tcp_sock *tp = tcp_sk(sk);
2374	u32 copied_seq = READ_ONCE(tp->copied_seq);
2375	u32 rcv_nxt = READ_ONCE(tp->rcv_nxt);
2376	int inq;
2377
2378	inq = rcv_nxt - copied_seq;
2379	if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) {
2380	lock_sock(sk);
2381	inq = tp->rcv_nxt - tp->copied_seq;
2382	release_sock(sk);
2383	}
2384	/* After receiving a FIN, tell the user-space to continue reading
2385	* by returning a non-zero inq.
2386	*/
2387	if (inq == 0 && sock_flag(sk, flag: SOCK_DONE))
2388	inq = 1;
2389	return inq;
2390	}
2391
2392	/* batch __xa_alloc() calls and reduce xa_lock()/xa_unlock() overhead. */
2393	struct tcp_xa_pool {
2394	u8 max; /* max <= MAX_SKB_FRAGS */
2395	u8 idx; /* idx <= max */
2396	__u32 tokens[MAX_SKB_FRAGS];
2397	netmem_ref netmems[MAX_SKB_FRAGS];
2398	};
2399
2400	static void tcp_xa_pool_commit_locked(struct sock *sk, struct tcp_xa_pool *p)
2401	{
2402	int i;
2403
2404	/* Commit part that has been copied to user space. */
2405	for (i = 0; i < p->idx; i++)
2406	__xa_cmpxchg(&sk->sk_user_frags, index: p->tokens[i], XA_ZERO_ENTRY,
2407	entry: (__force void *)p->netmems[i], GFP_KERNEL);
2408	/* Rollback what has been pre-allocated and is no longer needed. */
2409	for (; i < p->max; i++)
2410	__xa_erase(&sk->sk_user_frags, index: p->tokens[i]);
2411
2412	p->max = 0;
2413	p->idx = 0;
2414	}
2415
2416	static void tcp_xa_pool_commit(struct sock *sk, struct tcp_xa_pool *p)
2417	{
2418	if (!p->max)
2419	return;
2420
2421	xa_lock_bh(&sk->sk_user_frags);
2422
2423	tcp_xa_pool_commit_locked(sk, p);
2424
2425	xa_unlock_bh(&sk->sk_user_frags);
2426	}
2427
2428	static int tcp_xa_pool_refill(struct sock *sk, struct tcp_xa_pool *p,
2429	unsigned int max_frags)
2430	{
2431	int err, k;
2432
2433	if (p->idx < p->max)
2434	return 0;
2435
2436	xa_lock_bh(&sk->sk_user_frags);
2437
2438	tcp_xa_pool_commit_locked(sk, p);
2439
2440	for (k = 0; k < max_frags; k++) {
2441	err = __xa_alloc(&sk->sk_user_frags, id: &p->tokens[k],
2442	XA_ZERO_ENTRY, xa_limit_31b, GFP_KERNEL);
2443	if (err)
2444	break;
2445	}
2446
2447	xa_unlock_bh(&sk->sk_user_frags);
2448
2449	p->max = k;
2450	p->idx = 0;
2451	return k ? 0 : err;
2452	}
2453
2454	/* On error, returns the -errno. On success, returns number of bytes sent to the
2455	* user. May not consume all of @remaining_len.
2456	*/
2457	static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb,
2458	unsigned int offset, struct msghdr *msg,
2459	int remaining_len)
2460	{
2461	struct dmabuf_cmsg dmabuf_cmsg = { 0 };
2462	struct tcp_xa_pool tcp_xa_pool;
2463	unsigned int start;
2464	int i, copy, n;
2465	int sent = 0;
2466	int err = 0;
2467
2468	tcp_xa_pool.max = 0;
2469	tcp_xa_pool.idx = 0;
2470	do {
2471	start = skb_headlen(skb);
2472
2473	if (skb_frags_readable(skb)) {
2474	err = -ENODEV;
2475	goto out;
2476	}
2477
2478	/* Copy header. */
2479	copy = start - offset;
2480	if (copy > 0) {
2481	copy = min(copy, remaining_len);
2482
2483	n = copy_to_iter(addr: skb->data + offset, bytes: copy,
2484	i: &msg->msg_iter);
2485	if (n != copy) {
2486	err = -EFAULT;
2487	goto out;
2488	}
2489
2490	offset += copy;
2491	remaining_len -= copy;
2492
2493	/* First a dmabuf_cmsg for # bytes copied to user
2494	* buffer.
2495	*/
2496	memset(&dmabuf_cmsg, 0, sizeof(dmabuf_cmsg));
2497	dmabuf_cmsg.frag_size = copy;
2498	err = put_cmsg_notrunc(msg, SOL_SOCKET,
2499	SO_DEVMEM_LINEAR,
2500	len: sizeof(dmabuf_cmsg),
2501	data: &dmabuf_cmsg);
2502	if (err)
2503	goto out;
2504
2505	sent += copy;
2506
2507	if (remaining_len == 0)
2508	goto out;
2509	}
2510
2511	/* after that, send information of dmabuf pages through a
2512	* sequence of cmsg
2513	*/
2514	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2515	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2516	struct net_iov *niov;
2517	u64 frag_offset;
2518	int end;
2519
2520	/* !skb_frags_readable() should indicate that ALL the
2521	* frags in this skb are dmabuf net_iovs. We're checking
2522	* for that flag above, but also check individual frags
2523	* here. If the tcp stack is not setting
2524	* skb_frags_readable() correctly, we still don't want
2525	* to crash here.
2526	*/
2527	if (!skb_frag_net_iov(frag)) {
2528	net_err_ratelimited("Found non-dmabuf skb with net_iov");
2529	err = -ENODEV;
2530	goto out;
2531	}
2532
2533	niov = skb_frag_net_iov(frag);
2534	if (!net_is_devmem_iov(niov)) {
2535	err = -ENODEV;
2536	goto out;
2537	}
2538
2539	end = start + skb_frag_size(frag);
2540	copy = end - offset;
2541
2542	if (copy > 0) {
2543	copy = min(copy, remaining_len);
2544
2545	frag_offset = net_iov_virtual_addr(niov) +
2546	skb_frag_off(frag) + offset -
2547	start;
2548	dmabuf_cmsg.frag_offset = frag_offset;
2549	dmabuf_cmsg.frag_size = copy;
2550	err = tcp_xa_pool_refill(sk, p: &tcp_xa_pool,
2551	skb_shinfo(skb)->nr_frags - i);
2552	if (err)
2553	goto out;
2554
2555	/* Will perform the exchange later */
2556	dmabuf_cmsg.frag_token = tcp_xa_pool.tokens[tcp_xa_pool.idx];
2557	dmabuf_cmsg.dmabuf_id = net_devmem_iov_binding_id(niov);
2558
2559	offset += copy;
2560	remaining_len -= copy;
2561
2562	err = put_cmsg_notrunc(msg, SOL_SOCKET,
2563	SO_DEVMEM_DMABUF,
2564	len: sizeof(dmabuf_cmsg),
2565	data: &dmabuf_cmsg);
2566	if (err)
2567	goto out;
2568
2569	atomic_long_inc(v: &niov->pp_ref_count);
2570	tcp_xa_pool.netmems[tcp_xa_pool.idx++] = skb_frag_netmem(frag);
2571
2572	sent += copy;
2573
2574	if (remaining_len == 0)
2575	goto out;
2576	}
2577	start = end;
2578	}
2579
2580	tcp_xa_pool_commit(sk, p: &tcp_xa_pool);
2581	if (!remaining_len)
2582	goto out;
2583
2584	/* if remaining_len is not satisfied yet, we need to go to the
2585	* next frag in the frag_list to satisfy remaining_len.
2586	*/
2587	skb = skb_shinfo(skb)->frag_list ?: skb->next;
2588
2589	offset = offset - start;
2590	} while (skb);
2591
2592	if (remaining_len) {
2593	err = -EFAULT;
2594	goto out;
2595	}
2596
2597	out:
2598	tcp_xa_pool_commit(sk, p: &tcp_xa_pool);
2599	if (!sent)
2600	sent = err;
2601
2602	return sent;
2603	}
2604
2605	/*
2606	* This routine copies from a sock struct into the user buffer.
2607	*
2608	* Technical note: in 2.3 we work on _locked_ socket, so that
2609	* tricks with *seq access order and skb->users are not required.
2610	* Probably, code can be easily improved even more.
2611	*/
2612
2613	static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
2614	int flags, struct scm_timestamping_internal *tss,
2615	int *cmsg_flags)
2616	{
2617	struct tcp_sock *tp = tcp_sk(sk);
2618	int last_copied_dmabuf = -1; /* uninitialized */
2619	int copied = 0;
2620	u32 peek_seq;
2621	u32 *seq;
2622	unsigned long used;
2623	int err;
2624	int target; /* Read at least this many bytes */
2625	long timeo;
2626	struct sk_buff *skb, *last;
2627	u32 peek_offset = 0;
2628	u32 urg_hole = 0;
2629
2630	err = -ENOTCONN;
2631	if (sk->sk_state == TCP_LISTEN)
2632	goto out;
2633
2634	if (tp->recvmsg_inq) {
2635	*cmsg_flags = TCP_CMSG_INQ;
2636	msg->msg_get_inq = 1;
2637	}
2638	timeo = sock_rcvtimeo(sk, noblock: flags & MSG_DONTWAIT);
2639
2640	/* Urgent data needs to be handled specially. */
2641	if (flags & MSG_OOB)
2642	goto recv_urg;
2643
2644	if (unlikely(tp->repair)) {
2645	err = -EPERM;
2646	if (!(flags & MSG_PEEK))
2647	goto out;
2648
2649	if (tp->repair_queue == TCP_SEND_QUEUE)
2650	goto recv_sndq;
2651
2652	err = -EINVAL;
2653	if (tp->repair_queue == TCP_NO_QUEUE)
2654	goto out;
2655
2656	/* 'common' recv queue MSG_PEEK-ing */
2657	}
2658
2659	seq = &tp->copied_seq;
2660	if (flags & MSG_PEEK) {
2661	peek_offset = max(sk_peek_offset(sk, flags), 0);
2662	peek_seq = tp->copied_seq + peek_offset;
2663	seq = &peek_seq;
2664	}
2665
2666	target = sock_rcvlowat(sk, waitall: flags & MSG_WAITALL, len);
2667
2668	do {
2669	u32 offset;
2670
2671	/* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
2672	if (unlikely(tp->urg_data) && tp->urg_seq == *seq) {
2673	if (copied)
2674	break;
2675	if (signal_pending(current)) {
2676	copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
2677	break;
2678	}
2679	}
2680
2681	/* Next get a buffer. */
2682
2683	last = skb_peek_tail(list_: &sk->sk_receive_queue);
2684	skb_queue_walk(&sk->sk_receive_queue, skb) {
2685	last = skb;
2686	/* Now that we have two receive queues this
2687	* shouldn't happen.
2688	*/
2689	if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
2690	"TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
2691	*seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
2692	flags))
2693	break;
2694
2695	offset = *seq - TCP_SKB_CB(skb)->seq;
2696	if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
2697	pr_err_once("%s: found a SYN, please report !\n", __func__);
2698	offset--;
2699	}
2700	if (offset < skb->len)
2701	goto found_ok_skb;
2702	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2703	goto found_fin_ok;
2704	WARN(!(flags & MSG_PEEK),
2705	"TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
2706	*seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
2707	}
2708
2709	/* Well, if we have backlog, try to process it now yet. */
2710
2711	if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
2712	break;
2713
2714	if (copied) {
2715	if (!timeo ||
2716	sk->sk_err ||
2717	sk->sk_state == TCP_CLOSE ||
2718	(sk->sk_shutdown & RCV_SHUTDOWN) ||
2719	signal_pending(current))
2720	break;
2721	} else {
2722	if (sock_flag(sk, flag: SOCK_DONE))
2723	break;
2724
2725	if (sk->sk_err) {
2726	copied = sock_error(sk);
2727	break;
2728	}
2729
2730	if (sk->sk_shutdown & RCV_SHUTDOWN)
2731	break;
2732
2733	if (sk->sk_state == TCP_CLOSE) {
2734	/* This occurs when user tries to read
2735	* from never connected socket.
2736	*/
2737	copied = -ENOTCONN;
2738	break;
2739	}
2740
2741	if (!timeo) {
2742	copied = -EAGAIN;
2743	break;
2744	}
2745
2746	if (signal_pending(current)) {
2747	copied = sock_intr_errno(timeo);
2748	break;
2749	}
2750	}
2751
2752	if (copied >= target) {
2753	/* Do not sleep, just process backlog. */
2754	__sk_flush_backlog(sk);
2755	} else {
2756	tcp_cleanup_rbuf(sk, copied);
2757	err = sk_wait_data(sk, timeo: &timeo, skb: last);
2758	if (err < 0) {
2759	err = copied ? : err;
2760	goto out;
2761	}
2762	}
2763
2764	if ((flags & MSG_PEEK) &&
2765	(peek_seq - peek_offset - copied - urg_hole != tp->copied_seq)) {
2766	net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
2767	current->comm,
2768	task_pid_nr(current));
2769	peek_seq = tp->copied_seq + peek_offset;
2770	}
2771	continue;
2772
2773	found_ok_skb:
2774	/* Ok so how much can we use? */
2775	used = skb->len - offset;
2776	if (len < used)
2777	used = len;
2778
2779	/* Do we have urgent data here? */
2780	if (unlikely(tp->urg_data)) {
2781	u32 urg_offset = tp->urg_seq - *seq;
2782	if (urg_offset < used) {
2783	if (!urg_offset) {
2784	if (!sock_flag(sk, flag: SOCK_URGINLINE)) {
2785	WRITE_ONCE(*seq, *seq + 1);
2786	urg_hole++;
2787	offset++;
2788	used--;
2789	if (!used)
2790	goto skip_copy;
2791	}
2792	} else
2793	used = urg_offset;
2794	}
2795	}
2796
2797	if (!(flags & MSG_TRUNC)) {
2798	if (last_copied_dmabuf != -1 &&
2799	last_copied_dmabuf != !skb_frags_readable(skb))
2800	break;
2801
2802	if (skb_frags_readable(skb)) {
2803	err = skb_copy_datagram_msg(from: skb, offset, msg,
2804	size: used);
2805	if (err) {
2806	/* Exception. Bailout! */
2807	if (!copied)
2808	copied = -EFAULT;
2809	break;
2810	}
2811	} else {
2812	if (!(flags & MSG_SOCK_DEVMEM)) {
2813	/* dmabuf skbs can only be received
2814	* with the MSG_SOCK_DEVMEM flag.
2815	*/
2816	if (!copied)
2817	copied = -EFAULT;
2818
2819	break;
2820	}
2821
2822	err = tcp_recvmsg_dmabuf(sk, skb, offset, msg,
2823	remaining_len: used);
2824	if (err <= 0) {
2825	if (!copied)
2826	copied = -EFAULT;
2827
2828	break;
2829	}
2830	used = err;
2831	}
2832	}
2833
2834	last_copied_dmabuf = !skb_frags_readable(skb);
2835
2836	WRITE_ONCE(*seq, *seq + used);
2837	copied += used;
2838	len -= used;
2839	if (flags & MSG_PEEK)
2840	sk_peek_offset_fwd(sk, val: used);
2841	else
2842	sk_peek_offset_bwd(sk, val: used);
2843	tcp_rcv_space_adjust(sk);
2844
2845	skip_copy:
2846	if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) {
2847	WRITE_ONCE(tp->urg_data, 0);
2848	tcp_fast_path_check(sk);
2849	}
2850
2851	if (TCP_SKB_CB(skb)->has_rxtstamp) {
2852	tcp_update_recv_tstamps(skb, tss);
2853	*cmsg_flags |= TCP_CMSG_TS;
2854	}
2855
2856	if (used + offset < skb->len)
2857	continue;
2858
2859	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2860	goto found_fin_ok;
2861	if (!(flags & MSG_PEEK))
2862	tcp_eat_recv_skb(sk, skb);
2863	continue;
2864
2865	found_fin_ok:
2866	/* Process the FIN. */
2867	WRITE_ONCE(*seq, *seq + 1);
2868	if (!(flags & MSG_PEEK))
2869	tcp_eat_recv_skb(sk, skb);
2870	break;
2871	} while (len > 0);
2872
2873	/* According to UNIX98, msg_name/msg_namelen are ignored
2874	* on connected socket. I was just happy when found this 8) --ANK
2875	*/
2876
2877	/* Clean up data we have read: This will do ACK frames. */
2878	tcp_cleanup_rbuf(sk, copied);
2879	return copied;
2880
2881	out:
2882	return err;
2883
2884	recv_urg:
2885	err = tcp_recv_urg(sk, msg, len, flags);
2886	goto out;
2887
2888	recv_sndq:
2889	err = tcp_peek_sndq(sk, msg, len);
2890	goto out;
2891	}
2892
2893	int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags,
2894	int *addr_len)
2895	{
2896	int cmsg_flags = 0, ret;
2897	struct scm_timestamping_internal tss;
2898
2899	if (unlikely(flags & MSG_ERRQUEUE))
2900	return inet_recv_error(sk, msg, len, addr_len);
2901
2902	if (sk_can_busy_loop(sk) &&
2903	skb_queue_empty_lockless(list: &sk->sk_receive_queue) &&
2904	sk->sk_state == TCP_ESTABLISHED)
2905	sk_busy_loop(sk, nonblock: flags & MSG_DONTWAIT);
2906
2907	lock_sock(sk);
2908	ret = tcp_recvmsg_locked(sk, msg, len, flags, tss: &tss, cmsg_flags: &cmsg_flags);
2909	release_sock(sk);
2910
2911	if ((cmsg_flags || msg->msg_get_inq) && ret >= 0) {
2912	if (cmsg_flags & TCP_CMSG_TS)
2913	tcp_recv_timestamp(msg, sk, tss: &tss);
2914	if (msg->msg_get_inq) {
2915	msg->msg_inq = tcp_inq_hint(sk);
2916	if (cmsg_flags & TCP_CMSG_INQ)
2917	put_cmsg(msg, SOL_TCP, TCP_CM_INQ,
2918	len: sizeof(msg->msg_inq), data: &msg->msg_inq);
2919	}
2920	}
2921	return ret;
2922	}
2923	EXPORT_IPV6_MOD(tcp_recvmsg);
2924
2925	void tcp_set_state(struct sock *sk, int state)
2926	{
2927	int oldstate = sk->sk_state;
2928
2929	/* We defined a new enum for TCP states that are exported in BPF
2930	* so as not force the internal TCP states to be frozen. The
2931	* following checks will detect if an internal state value ever
2932	* differs from the BPF value. If this ever happens, then we will
2933	* need to remap the internal value to the BPF value before calling
2934	* tcp_call_bpf_2arg.
2935	*/
2936	BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED);
2937	BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT);
2938	BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV);
2939	BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1);
2940	BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2);
2941	BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT);
2942	BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE);
2943	BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT);
2944	BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK);
2945	BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN);
2946	BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING);
2947	BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV);
2948	BUILD_BUG_ON((int)BPF_TCP_BOUND_INACTIVE != (int)TCP_BOUND_INACTIVE);
2949	BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES);
2950
2951	/* bpf uapi header bpf.h defines an anonymous enum with values
2952	* BPF_TCP_* used by bpf programs. Currently gcc built vmlinux
2953	* is able to emit this enum in DWARF due to the above BUILD_BUG_ON.
2954	* But clang built vmlinux does not have this enum in DWARF
2955	* since clang removes the above code before generating IR/debuginfo.
2956	* Let us explicitly emit the type debuginfo to ensure the
2957	* above-mentioned anonymous enum in the vmlinux DWARF and hence BTF
2958	* regardless of which compiler is used.
2959	*/
2960	BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED);
2961
2962	if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
2963	tcp_call_bpf_2arg(sk, op: BPF_SOCK_OPS_STATE_CB, arg1: oldstate, arg2: state);
2964
2965	switch (state) {
2966	case TCP_ESTABLISHED:
2967	if (oldstate != TCP_ESTABLISHED)
2968	TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2969	break;
2970	case TCP_CLOSE_WAIT:
2971	if (oldstate == TCP_SYN_RECV)
2972	TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2973	break;
2974
2975	case TCP_CLOSE:
2976	if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
2977	TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
2978
2979	sk->sk_prot->unhash(sk);
2980	if (inet_csk(sk)->icsk_bind_hash &&
2981	!(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
2982	inet_put_port(sk);
2983	fallthrough;
2984	default:
2985	if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT)
2986	TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2987	}
2988
2989	/* Change state AFTER socket is unhashed to avoid closed
2990	* socket sitting in hash tables.
2991	*/
2992	inet_sk_state_store(sk, newstate: state);
2993	}
2994	EXPORT_SYMBOL_GPL(tcp_set_state);
2995
2996	/*
2997	* State processing on a close. This implements the state shift for
2998	* sending our FIN frame. Note that we only send a FIN for some
2999	* states. A shutdown() may have already sent the FIN, or we may be
3000	* closed.
3001	*/
3002
3003	static const unsigned char new_state[16] = {
3004	/* current state: new state: action: */
3005	[0 /* (Invalid) */] = TCP_CLOSE,
3006	[TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
3007	[TCP_SYN_SENT] = TCP_CLOSE,
3008	[TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
3009	[TCP_FIN_WAIT1] = TCP_FIN_WAIT1,
3010	[TCP_FIN_WAIT2] = TCP_FIN_WAIT2,
3011	[TCP_TIME_WAIT] = TCP_CLOSE,
3012	[TCP_CLOSE] = TCP_CLOSE,
3013	[TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN,
3014	[TCP_LAST_ACK] = TCP_LAST_ACK,
3015	[TCP_LISTEN] = TCP_CLOSE,
3016	[TCP_CLOSING] = TCP_CLOSING,
3017	[TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */
3018	};
3019
3020	static int tcp_close_state(struct sock *sk)
3021	{
3022	int next = (int)new_state[sk->sk_state];
3023	int ns = next & TCP_STATE_MASK;
3024
3025	tcp_set_state(sk, ns);
3026
3027	return next & TCP_ACTION_FIN;
3028	}
3029
3030	/*
3031	* Shutdown the sending side of a connection. Much like close except
3032	* that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
3033	*/
3034
3035	void tcp_shutdown(struct sock *sk, int how)
3036	{
3037	/* We need to grab some memory, and put together a FIN,
3038	* and then put it into the queue to be sent.
3039	* Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
3040	*/
3041	if (!(how & SEND_SHUTDOWN))
3042	return;
3043
3044	/* If we've already sent a FIN, or it's a closed state, skip this. */
3045	if ((1 << sk->sk_state) &
3046	(TCPF_ESTABLISHED | TCPF_SYN_SENT |
3047	TCPF_CLOSE_WAIT)) {
3048	/* Clear out any half completed packets. FIN if needed. */
3049	if (tcp_close_state(sk))
3050	tcp_send_fin(sk);
3051	}
3052	}
3053	EXPORT_IPV6_MOD(tcp_shutdown);
3054
3055	int tcp_orphan_count_sum(void)
3056	{
3057	int i, total = 0;
3058
3059	for_each_possible_cpu(i)
3060	total += per_cpu(tcp_orphan_count, i);
3061
3062	return max(total, 0);
3063	}
3064
3065	static int tcp_orphan_cache;
3066	static struct timer_list tcp_orphan_timer;
3067	#define TCP_ORPHAN_TIMER_PERIOD msecs_to_jiffies(100)
3068
3069	static void tcp_orphan_update(struct timer_list *unused)
3070	{
3071	WRITE_ONCE(tcp_orphan_cache, tcp_orphan_count_sum());
3072	mod_timer(timer: &tcp_orphan_timer, expires: jiffies + TCP_ORPHAN_TIMER_PERIOD);
3073	}
3074
3075	static bool tcp_too_many_orphans(int shift)
3076	{
3077	return READ_ONCE(tcp_orphan_cache) << shift >
3078	READ_ONCE(sysctl_tcp_max_orphans);
3079	}
3080
3081	static bool tcp_out_of_memory(const struct sock *sk)
3082	{
3083	if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
3084	sk_memory_allocated(sk) > sk_prot_mem_limits(sk, index: 2))
3085	return true;
3086	return false;
3087	}
3088
3089	bool tcp_check_oom(const struct sock *sk, int shift)
3090	{
3091	bool too_many_orphans, out_of_socket_memory;
3092
3093	too_many_orphans = tcp_too_many_orphans(shift);
3094	out_of_socket_memory = tcp_out_of_memory(sk);
3095
3096	if (too_many_orphans)
3097	net_info_ratelimited("too many orphaned sockets\n");
3098	if (out_of_socket_memory)
3099	net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
3100	return too_many_orphans || out_of_socket_memory;
3101	}
3102
3103	void __tcp_close(struct sock *sk, long timeout)
3104	{
3105	struct sk_buff *skb;
3106	int data_was_unread = 0;
3107	int state;
3108
3109	WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
3110
3111	if (sk->sk_state == TCP_LISTEN) {
3112	tcp_set_state(sk, TCP_CLOSE);
3113
3114	/* Special case. */
3115	inet_csk_listen_stop(sk);
3116
3117	goto adjudge_to_death;
3118	}
3119
3120	/* We need to flush the recv. buffs. We do this only on the
3121	* descriptor close, not protocol-sourced closes, because the
3122	* reader process may not have drained the data yet!
3123	*/
3124	while ((skb = __skb_dequeue(list: &sk->sk_receive_queue)) != NULL) {
3125	u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
3126
3127	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
3128	len--;
3129	data_was_unread += len;
3130	__kfree_skb(skb);
3131	}
3132
3133	/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
3134	if (sk->sk_state == TCP_CLOSE)
3135	goto adjudge_to_death;
3136
3137	/* As outlined in RFC 2525, section 2.17, we send a RST here because
3138	* data was lost. To witness the awful effects of the old behavior of
3139	* always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
3140	* GET in an FTP client, suspend the process, wait for the client to
3141	* advertise a zero window, then kill -9 the FTP client, wheee...
3142	* Note: timeout is always zero in such a case.
3143	*/
3144	if (unlikely(tcp_sk(sk)->repair)) {
3145	sk->sk_prot->disconnect(sk, 0);
3146	} else if (data_was_unread) {
3147	/* Unread data was tossed, zap the connection. */
3148	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
3149	tcp_set_state(sk, TCP_CLOSE);
3150	tcp_send_active_reset(sk, priority: sk->sk_allocation,
3151	reason: SK_RST_REASON_TCP_ABORT_ON_CLOSE);
3152	} else if (sock_flag(sk, flag: SOCK_LINGER) && !sk->sk_lingertime) {
3153	/* Check zero linger _after_ checking for unread data. */
3154	sk->sk_prot->disconnect(sk, 0);
3155	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
3156	} else if (tcp_close_state(sk)) {
3157	/* We FIN if the application ate all the data before
3158	* zapping the connection.
3159	*/
3160
3161	/* RED-PEN. Formally speaking, we have broken TCP state
3162	* machine. State transitions:
3163	*
3164	* TCP_ESTABLISHED -> TCP_FIN_WAIT1
3165	* TCP_SYN_RECV -> TCP_FIN_WAIT1 (it is difficult)
3166	* TCP_CLOSE_WAIT -> TCP_LAST_ACK
3167	*
3168	* are legal only when FIN has been sent (i.e. in window),
3169	* rather than queued out of window. Purists blame.
3170	*
3171	* F.e. "RFC state" is ESTABLISHED,
3172	* if Linux state is FIN-WAIT-1, but FIN is still not sent.
3173	*
3174	* The visible declinations are that sometimes
3175	* we enter time-wait state, when it is not required really
3176	* (harmless), do not send active resets, when they are
3177	* required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
3178	* they look as CLOSING or LAST_ACK for Linux)
3179	* Probably, I missed some more holelets.
3180	* --ANK
3181	* XXX (TFO) - To start off we don't support SYN+ACK+FIN
3182	* in a single packet! (May consider it later but will
3183	* probably need API support or TCP_CORK SYN-ACK until
3184	* data is written and socket is closed.)
3185	*/
3186	tcp_send_fin(sk);
3187	}
3188
3189	sk_stream_wait_close(sk, timeo_p: timeout);
3190
3191	adjudge_to_death:
3192	state = sk->sk_state;
3193	sock_hold(sk);
3194	sock_orphan(sk);
3195
3196	local_bh_disable();
3197	bh_lock_sock(sk);
3198	/* remove backlog if any, without releasing ownership. */
3199	__release_sock(sk);
3200
3201	this_cpu_inc(tcp_orphan_count);
3202
3203	/* Have we already been destroyed by a softirq or backlog? */
3204	if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
3205	goto out;
3206
3207	/* This is a (useful) BSD violating of the RFC. There is a
3208	* problem with TCP as specified in that the other end could
3209	* keep a socket open forever with no application left this end.
3210	* We use a 1 minute timeout (about the same as BSD) then kill
3211	* our end. If they send after that then tough - BUT: long enough
3212	* that we won't make the old 4*rto = almost no time - whoops
3213	* reset mistake.
3214	*
3215	* Nope, it was not mistake. It is really desired behaviour
3216	* f.e. on http servers, when such sockets are useless, but
3217	* consume significant resources. Let's do it with special
3218	* linger2 option. --ANK
3219	*/
3220
3221	if (sk->sk_state == TCP_FIN_WAIT2) {
3222	struct tcp_sock *tp = tcp_sk(sk);
3223	if (READ_ONCE(tp->linger2) < 0) {
3224	tcp_set_state(sk, TCP_CLOSE);
3225	tcp_send_active_reset(sk, GFP_ATOMIC,
3226	reason: SK_RST_REASON_TCP_ABORT_ON_LINGER);
3227	__NET_INC_STATS(sock_net(sk),
3228	LINUX_MIB_TCPABORTONLINGER);
3229	} else {
3230	const int tmo = tcp_fin_time(sk);
3231
3232	if (tmo > TCP_TIMEWAIT_LEN) {
3233	tcp_reset_keepalive_timer(sk,
3234	timeout: tmo - TCP_TIMEWAIT_LEN);
3235	} else {
3236	tcp_time_wait(sk, state: TCP_FIN_WAIT2, timeo: tmo);
3237	goto out;
3238	}
3239	}
3240	}
3241	if (sk->sk_state != TCP_CLOSE) {
3242	if (tcp_check_oom(sk, shift: 0)) {
3243	tcp_set_state(sk, TCP_CLOSE);
3244	tcp_send_active_reset(sk, GFP_ATOMIC,
3245	reason: SK_RST_REASON_TCP_ABORT_ON_MEMORY);
3246	__NET_INC_STATS(sock_net(sk),
3247	LINUX_MIB_TCPABORTONMEMORY);
3248	} else if (!check_net(net: sock_net(sk))) {
3249	/* Not possible to send reset; just close */
3250	tcp_set_state(sk, TCP_CLOSE);
3251	}
3252	}
3253
3254	if (sk->sk_state == TCP_CLOSE) {
3255	struct request_sock *req;
3256
3257	req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
3258	lockdep_sock_is_held(sk));
3259	/* We could get here with a non-NULL req if the socket is
3260	* aborted (e.g., closed with unread data) before 3WHS
3261	* finishes.
3262	*/
3263	if (req)
3264	reqsk_fastopen_remove(sk, req, reset: false);
3265	inet_csk_destroy_sock(sk);
3266	}
3267	/* Otherwise, socket is reprieved until protocol close. */
3268
3269	out:
3270	bh_unlock_sock(sk);
3271	local_bh_enable();
3272	}
3273
3274	void tcp_close(struct sock *sk, long timeout)
3275	{
3276	lock_sock(sk);
3277	__tcp_close(sk, timeout);
3278	release_sock(sk);
3279	if (!sk->sk_net_refcnt)
3280	inet_csk_clear_xmit_timers_sync(sk);
3281	sock_put(sk);
3282	}
3283	EXPORT_SYMBOL(tcp_close);
3284
3285	/* These states need RST on ABORT according to RFC793 */
3286
3287	static inline bool tcp_need_reset(int state)
3288	{
3289	return (1 << state) &
3290	(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
3291	TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
3292	}
3293
3294	static void tcp_rtx_queue_purge(struct sock *sk)
3295	{
3296	struct rb_node *p = rb_first(&sk->tcp_rtx_queue);
3297
3298	tcp_sk(sk)->highest_sack = NULL;
3299	while (p) {
3300	struct sk_buff *skb = rb_to_skb(p);
3301
3302	p = rb_next(p);
3303	/* Since we are deleting whole queue, no need to
3304	* list_del(&skb->tcp_tsorted_anchor)
3305	*/
3306	tcp_rtx_queue_unlink(skb, sk);
3307	tcp_wmem_free_skb(sk, skb);
3308	}
3309	}
3310
3311	void tcp_write_queue_purge(struct sock *sk)
3312	{
3313	struct sk_buff *skb;
3314
3315	tcp_chrono_stop(sk, type: TCP_CHRONO_BUSY);
3316	while ((skb = __skb_dequeue(list: &sk->sk_write_queue)) != NULL) {
3317	tcp_skb_tsorted_anchor_cleanup(skb);
3318	tcp_wmem_free_skb(sk, skb);
3319	}
3320	tcp_rtx_queue_purge(sk);
3321	INIT_LIST_HEAD(list: &tcp_sk(sk)->tsorted_sent_queue);
3322	tcp_clear_all_retrans_hints(tcp_sk(sk));
3323	tcp_sk(sk)->packets_out = 0;
3324	inet_csk(sk)->icsk_backoff = 0;
3325	}
3326
3327	int tcp_disconnect(struct sock *sk, int flags)
3328	{
3329	struct inet_sock *inet = inet_sk(sk);
3330	struct inet_connection_sock *icsk = inet_csk(sk);
3331	struct tcp_sock *tp = tcp_sk(sk);
3332	int old_state = sk->sk_state;
3333	u32 seq;
3334
3335	if (old_state != TCP_CLOSE)
3336	tcp_set_state(sk, TCP_CLOSE);
3337
3338	/* ABORT function of RFC793 */
3339	if (old_state == TCP_LISTEN) {
3340	inet_csk_listen_stop(sk);
3341	} else if (unlikely(tp->repair)) {
3342	WRITE_ONCE(sk->sk_err, ECONNABORTED);
3343	} else if (tcp_need_reset(state: old_state)) {
3344	tcp_send_active_reset(sk, priority: gfp_any(), reason: SK_RST_REASON_TCP_STATE);
3345	WRITE_ONCE(sk->sk_err, ECONNRESET);
3346	} else if (tp->snd_nxt != tp->write_seq &&
3347	(1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) {
3348	/* The last check adjusts for discrepancy of Linux wrt. RFC
3349	* states
3350	*/
3351	tcp_send_active_reset(sk, priority: gfp_any(),
3352	reason: SK_RST_REASON_TCP_DISCONNECT_WITH_DATA);
3353	WRITE_ONCE(sk->sk_err, ECONNRESET);
3354	} else if (old_state == TCP_SYN_SENT)
3355	WRITE_ONCE(sk->sk_err, ECONNRESET);
3356
3357	tcp_clear_xmit_timers(sk);
3358	__skb_queue_purge(list: &sk->sk_receive_queue);
3359	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
3360	WRITE_ONCE(tp->urg_data, 0);
3361	sk_set_peek_off(sk, val: -1);
3362	tcp_write_queue_purge(sk);
3363	tcp_fastopen_active_disable_ofo_check(sk);
3364	skb_rbtree_purge(root: &tp->out_of_order_queue);
3365
3366	inet->inet_dport = 0;
3367
3368	inet_bhash2_reset_saddr(sk);
3369
3370	WRITE_ONCE(sk->sk_shutdown, 0);
3371	sock_reset_flag(sk, flag: SOCK_DONE);
3372	tp->srtt_us = 0;
3373	tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
3374	tp->rcv_rtt_last_tsecr = 0;
3375
3376	seq = tp->write_seq + tp->max_window + 2;
3377	if (!seq)
3378	seq = 1;
3379	WRITE_ONCE(tp->write_seq, seq);
3380
3381	icsk->icsk_backoff = 0;
3382	icsk->icsk_probes_out = 0;
3383	icsk->icsk_probes_tstamp = 0;
3384	icsk->icsk_rto = TCP_TIMEOUT_INIT;
3385	WRITE_ONCE(icsk->icsk_rto_min, TCP_RTO_MIN);
3386	WRITE_ONCE(icsk->icsk_delack_max, TCP_DELACK_MAX);
3387	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
3388	tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
3389	tp->snd_cwnd_cnt = 0;
3390	tp->is_cwnd_limited = 0;
3391	tp->max_packets_out = 0;
3392	tp->window_clamp = 0;
3393	tp->delivered = 0;
3394	tp->delivered_ce = 0;
3395	if (icsk->icsk_ca_initialized && icsk->icsk_ca_ops->release)
3396	icsk->icsk_ca_ops->release(sk);
3397	memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
3398	icsk->icsk_ca_initialized = 0;
3399	tcp_set_ca_state(sk, ca_state: TCP_CA_Open);
3400	tp->is_sack_reneg = 0;
3401	tcp_clear_retrans(tp);
3402	tp->total_retrans = 0;
3403	inet_csk_delack_init(sk);
3404	/* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0
3405	* issue in __tcp_select_window()
3406	*/
3407	icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
3408	memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
3409	__sk_dst_reset(sk);
3410	dst_release(unrcu_pointer(xchg(&sk->sk_rx_dst, NULL)));
3411	tcp_saved_syn_free(tp);
3412	tp->compressed_ack = 0;
3413	tp->segs_in = 0;
3414	tp->segs_out = 0;
3415	tp->bytes_sent = 0;
3416	tp->bytes_acked = 0;
3417	tp->bytes_received = 0;
3418	tp->bytes_retrans = 0;
3419	tp->data_segs_in = 0;
3420	tp->data_segs_out = 0;
3421	tp->duplicate_sack[0].start_seq = 0;
3422	tp->duplicate_sack[0].end_seq = 0;
3423	tp->dsack_dups = 0;
3424	tp->reord_seen = 0;
3425	tp->retrans_out = 0;
3426	tp->sacked_out = 0;
3427	tp->tlp_high_seq = 0;
3428	tp->last_oow_ack_time = 0;
3429	tp->plb_rehash = 0;
3430	/* There's a bubble in the pipe until at least the first ACK. */
3431	tp->app_limited = ~0U;
3432	tp->rate_app_limited = 1;
3433	tp->rack.mstamp = 0;
3434	tp->rack.advanced = 0;
3435	tp->rack.reo_wnd_steps = 1;
3436	tp->rack.last_delivered = 0;
3437	tp->rack.reo_wnd_persist = 0;
3438	tp->rack.dsack_seen = 0;
3439	tp->syn_data_acked = 0;
3440	tp->syn_fastopen_child = 0;
3441	tp->rx_opt.saw_tstamp = 0;
3442	tp->rx_opt.dsack = 0;
3443	tp->rx_opt.num_sacks = 0;
3444	tp->rcv_ooopack = 0;
3445
3446
3447	/* Clean up fastopen related fields */
3448	tcp_free_fastopen_req(tp);
3449	inet_clear_bit(DEFER_CONNECT, sk);
3450	tp->fastopen_client_fail = 0;
3451
3452	WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
3453
3454	if (sk->sk_frag.page) {
3455	put_page(page: sk->sk_frag.page);
3456	sk->sk_frag.page = NULL;
3457	sk->sk_frag.offset = 0;
3458	}
3459	sk_error_report(sk);
3460	return 0;
3461	}
3462	EXPORT_SYMBOL(tcp_disconnect);
3463
3464	static inline bool tcp_can_repair_sock(const struct sock *sk)
3465	{
3466	return sockopt_ns_capable(ns: sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
3467	(sk->sk_state != TCP_LISTEN);
3468	}
3469
3470	static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len)
3471	{
3472	struct tcp_repair_window opt;
3473
3474	if (!tp->repair)
3475	return -EPERM;
3476
3477	if (len != sizeof(opt))
3478	return -EINVAL;
3479
3480	if (copy_from_sockptr(dst: &opt, src: optbuf, size: sizeof(opt)))
3481	return -EFAULT;
3482
3483	if (opt.max_window < opt.snd_wnd)
3484	return -EINVAL;
3485
3486	if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))
3487	return -EINVAL;
3488
3489	if (after(opt.rcv_wup, tp->rcv_nxt))
3490	return -EINVAL;
3491
3492	tp->snd_wl1 = opt.snd_wl1;
3493	tp->snd_wnd = opt.snd_wnd;
3494	tp->max_window = opt.max_window;
3495
3496	tp->rcv_wnd = opt.rcv_wnd;
3497	tp->rcv_wup = opt.rcv_wup;
3498
3499	return 0;
3500	}
3501
3502	static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf,
3503	unsigned int len)
3504	{
3505	struct tcp_sock *tp = tcp_sk(sk);
3506	struct tcp_repair_opt opt;
3507	size_t offset = 0;
3508
3509	while (len >= sizeof(opt)) {
3510	if (copy_from_sockptr_offset(dst: &opt, src: optbuf, offset, size: sizeof(opt)))
3511	return -EFAULT;
3512
3513	offset += sizeof(opt);
3514	len -= sizeof(opt);
3515
3516	switch (opt.opt_code) {
3517	case TCPOPT_MSS:
3518	tp->rx_opt.mss_clamp = opt.opt_val;
3519	tcp_mtup_init(sk);
3520	break;
3521	case TCPOPT_WINDOW:
3522	{
3523	u16 snd_wscale = opt.opt_val & 0xFFFF;
3524	u16 rcv_wscale = opt.opt_val >> 16;
3525
3526	if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE)
3527	return -EFBIG;
3528
3529	tp->rx_opt.snd_wscale = snd_wscale;
3530	tp->rx_opt.rcv_wscale = rcv_wscale;
3531	tp->rx_opt.wscale_ok = 1;
3532	}
3533	break;
3534	case TCPOPT_SACK_PERM:
3535	if (opt.opt_val != 0)
3536	return -EINVAL;
3537
3538	tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
3539	break;
3540	case TCPOPT_TIMESTAMP:
3541	if (opt.opt_val != 0)
3542	return -EINVAL;
3543
3544	tp->rx_opt.tstamp_ok = 1;
3545	break;
3546	}
3547	}
3548
3549	return 0;
3550	}
3551
3552	DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
3553	EXPORT_IPV6_MOD(tcp_tx_delay_enabled);
3554
3555	static void tcp_enable_tx_delay(void)
3556	{
3557	if (!static_branch_unlikely(&tcp_tx_delay_enabled)) {
3558	static int __tcp_tx_delay_enabled = 0;
3559
3560	if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) {
3561	static_branch_enable(&tcp_tx_delay_enabled);
3562	pr_info("TCP_TX_DELAY enabled\n");
3563	}
3564	}
3565	}
3566
3567	/* When set indicates to always queue non-full frames. Later the user clears
3568	* this option and we transmit any pending partial frames in the queue. This is
3569	* meant to be used alongside sendfile() to get properly filled frames when the
3570	* user (for example) must write out headers with a write() call first and then
3571	* use sendfile to send out the data parts.
3572	*
3573	* TCP_CORK can be set together with TCP_NODELAY and it is stronger than
3574	* TCP_NODELAY.
3575	*/
3576	void __tcp_sock_set_cork(struct sock *sk, bool on)
3577	{
3578	struct tcp_sock *tp = tcp_sk(sk);
3579
3580	if (on) {
3581	tp->nonagle |= TCP_NAGLE_CORK;
3582	} else {
3583	tp->nonagle &= ~TCP_NAGLE_CORK;
3584	if (tp->nonagle & TCP_NAGLE_OFF)
3585	tp->nonagle |= TCP_NAGLE_PUSH;
3586	tcp_push_pending_frames(sk);
3587	}
3588	}
3589
3590	void tcp_sock_set_cork(struct sock *sk, bool on)
3591	{
3592	lock_sock(sk);
3593	__tcp_sock_set_cork(sk, on);
3594	release_sock(sk);
3595	}
3596	EXPORT_SYMBOL(tcp_sock_set_cork);
3597
3598	/* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is
3599	* remembered, but it is not activated until cork is cleared.
3600	*
3601	* However, when TCP_NODELAY is set we make an explicit push, which overrides
3602	* even TCP_CORK for currently queued segments.
3603	*/
3604	void __tcp_sock_set_nodelay(struct sock *sk, bool on)
3605	{
3606	if (on) {
3607	tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
3608	tcp_push_pending_frames(sk);
3609	} else {
3610	tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF;
3611	}
3612	}
3613
3614	void tcp_sock_set_nodelay(struct sock *sk)
3615	{
3616	lock_sock(sk);
3617	__tcp_sock_set_nodelay(sk, on: true);
3618	release_sock(sk);
3619	}
3620	EXPORT_SYMBOL(tcp_sock_set_nodelay);
3621
3622	static void __tcp_sock_set_quickack(struct sock *sk, int val)
3623	{
3624	if (!val) {
3625	inet_csk_enter_pingpong_mode(sk);
3626	return;
3627	}
3628
3629	inet_csk_exit_pingpong_mode(sk);
3630	if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
3631	inet_csk_ack_scheduled(sk)) {
3632	inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED;
3633	tcp_cleanup_rbuf(sk, copied: 1);
3634	if (!(val & 1))
3635	inet_csk_enter_pingpong_mode(sk);
3636	}
3637	}
3638
3639	void tcp_sock_set_quickack(struct sock *sk, int val)
3640	{
3641	lock_sock(sk);
3642	__tcp_sock_set_quickack(sk, val);
3643	release_sock(sk);
3644	}
3645	EXPORT_SYMBOL(tcp_sock_set_quickack);
3646
3647	int tcp_sock_set_syncnt(struct sock *sk, int val)
3648	{
3649	if (val < 1 || val > MAX_TCP_SYNCNT)
3650	return -EINVAL;
3651
3652	WRITE_ONCE(inet_csk(sk)->icsk_syn_retries, val);
3653	return 0;
3654	}
3655	EXPORT_SYMBOL(tcp_sock_set_syncnt);
3656
3657	int tcp_sock_set_user_timeout(struct sock *sk, int val)
3658	{
3659	/* Cap the max time in ms TCP will retry or probe the window
3660	* before giving up and aborting (ETIMEDOUT) a connection.
3661	*/
3662	if (val < 0)
3663	return -EINVAL;
3664
3665	WRITE_ONCE(inet_csk(sk)->icsk_user_timeout, val);
3666	return 0;
3667	}
3668	EXPORT_SYMBOL(tcp_sock_set_user_timeout);
3669
3670	int tcp_sock_set_keepidle_locked(struct sock *sk, int val)
3671	{
3672	struct tcp_sock *tp = tcp_sk(sk);
3673
3674	if (val < 1 || val > MAX_TCP_KEEPIDLE)
3675	return -EINVAL;
3676
3677	/* Paired with WRITE_ONCE() in keepalive_time_when() */
3678	WRITE_ONCE(tp->keepalive_time, val * HZ);
3679	if (sock_flag(sk, flag: SOCK_KEEPOPEN) &&
3680	!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
3681	u32 elapsed = keepalive_time_elapsed(tp);
3682
3683	if (tp->keepalive_time > elapsed)
3684	elapsed = tp->keepalive_time - elapsed;
3685	else
3686	elapsed = 0;
3687	tcp_reset_keepalive_timer(sk, timeout: elapsed);
3688	}
3689
3690	return 0;
3691	}
3692
3693	int tcp_sock_set_keepidle(struct sock *sk, int val)
3694	{
3695	int err;
3696
3697	lock_sock(sk);
3698	err = tcp_sock_set_keepidle_locked(sk, val);
3699	release_sock(sk);
3700	return err;
3701	}
3702	EXPORT_SYMBOL(tcp_sock_set_keepidle);
3703
3704	int tcp_sock_set_keepintvl(struct sock *sk, int val)
3705	{
3706	if (val < 1 || val > MAX_TCP_KEEPINTVL)
3707	return -EINVAL;
3708
3709	WRITE_ONCE(tcp_sk(sk)->keepalive_intvl, val * HZ);
3710	return 0;
3711	}
3712	EXPORT_SYMBOL(tcp_sock_set_keepintvl);
3713
3714	int tcp_sock_set_keepcnt(struct sock *sk, int val)
3715	{
3716	if (val < 1 || val > MAX_TCP_KEEPCNT)
3717	return -EINVAL;
3718
3719	/* Paired with READ_ONCE() in keepalive_probes() */
3720	WRITE_ONCE(tcp_sk(sk)->keepalive_probes, val);
3721	return 0;
3722	}
3723	EXPORT_SYMBOL(tcp_sock_set_keepcnt);
3724
3725	int tcp_set_window_clamp(struct sock *sk, int val)
3726	{
3727	u32 old_window_clamp, new_window_clamp, new_rcv_ssthresh;
3728	struct tcp_sock *tp = tcp_sk(sk);
3729
3730	if (!val) {
3731	if (sk->sk_state != TCP_CLOSE)
3732	return -EINVAL;
3733	WRITE_ONCE(tp->window_clamp, 0);
3734	return 0;
3735	}
3736
3737	old_window_clamp = tp->window_clamp;
3738	new_window_clamp = max_t(int, SOCK_MIN_RCVBUF / 2, val);
3739
3740	if (new_window_clamp == old_window_clamp)
3741	return 0;
3742
3743	WRITE_ONCE(tp->window_clamp, new_window_clamp);
3744
3745	/* Need to apply the reserved mem provisioning only
3746	* when shrinking the window clamp.
3747	*/
3748	if (new_window_clamp < old_window_clamp) {
3749	__tcp_adjust_rcv_ssthresh(sk, new_ssthresh: new_window_clamp);
3750	} else {
3751	new_rcv_ssthresh = min(tp->rcv_wnd, new_window_clamp);
3752	tp->rcv_ssthresh = max(new_rcv_ssthresh, tp->rcv_ssthresh);
3753	}
3754	return 0;
3755	}
3756
3757	/*
3758	* Socket option code for TCP.
3759	*/
3760	int do_tcp_setsockopt(struct sock *sk, int level, int optname,
3761	sockptr_t optval, unsigned int optlen)
3762	{
3763	struct tcp_sock *tp = tcp_sk(sk);
3764	struct inet_connection_sock *icsk = inet_csk(sk);
3765	struct net *net = sock_net(sk);
3766	int val;
3767	int err = 0;
3768
3769	/* These are data/string values, all the others are ints */
3770	switch (optname) {
3771	case TCP_CONGESTION: {
3772	char name[TCP_CA_NAME_MAX];
3773
3774	if (optlen < 1)
3775	return -EINVAL;
3776
3777	val = strncpy_from_sockptr(dst: name, src: optval,
3778	min_t(long, TCP_CA_NAME_MAX-1, optlen));
3779	if (val < 0)
3780	return -EFAULT;
3781	name[val] = 0;
3782
3783	sockopt_lock_sock(sk);
3784	err = tcp_set_congestion_control(sk, name, load: !has_current_bpf_ctx(),
3785	cap_net_admin: sockopt_ns_capable(ns: sock_net(sk)->user_ns,
3786	CAP_NET_ADMIN));
3787	sockopt_release_sock(sk);
3788	return err;
3789	}
3790	case TCP_ULP: {
3791	char name[TCP_ULP_NAME_MAX];
3792
3793	if (optlen < 1)
3794	return -EINVAL;
3795
3796	val = strncpy_from_sockptr(dst: name, src: optval,
3797	min_t(long, TCP_ULP_NAME_MAX - 1,
3798	optlen));
3799	if (val < 0)
3800	return -EFAULT;
3801	name[val] = 0;
3802
3803	sockopt_lock_sock(sk);
3804	err = tcp_set_ulp(sk, name);
3805	sockopt_release_sock(sk);
3806	return err;
3807	}
3808	case TCP_FASTOPEN_KEY: {
3809	__u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
3810	__u8 *backup_key = NULL;
3811
3812	/* Allow a backup key as well to facilitate key rotation
3813	* First key is the active one.
3814	*/
3815	if (optlen != TCP_FASTOPEN_KEY_LENGTH &&
3816	optlen != TCP_FASTOPEN_KEY_BUF_LENGTH)
3817	return -EINVAL;
3818
3819	if (copy_from_sockptr(dst: key, src: optval, size: optlen))
3820	return -EFAULT;
3821
3822	if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH)
3823	backup_key = key + TCP_FASTOPEN_KEY_LENGTH;
3824
3825	return tcp_fastopen_reset_cipher(net, sk, primary_key: key, backup_key);
3826	}
3827	default:
3828	/* fallthru */
3829	break;
3830	}
3831
3832	if (optlen < sizeof(int))
3833	return -EINVAL;
3834
3835	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
3836	return -EFAULT;
3837
3838	/* Handle options that can be set without locking the socket. */
3839	switch (optname) {
3840	case TCP_SYNCNT:
3841	return tcp_sock_set_syncnt(sk, val);
3842	case TCP_USER_TIMEOUT:
3843	return tcp_sock_set_user_timeout(sk, val);
3844	case TCP_KEEPINTVL:
3845	return tcp_sock_set_keepintvl(sk, val);
3846	case TCP_KEEPCNT:
3847	return tcp_sock_set_keepcnt(sk, val);
3848	case TCP_LINGER2:
3849	if (val < 0)
3850	WRITE_ONCE(tp->linger2, -1);
3851	else if (val > TCP_FIN_TIMEOUT_MAX / HZ)
3852	WRITE_ONCE(tp->linger2, TCP_FIN_TIMEOUT_MAX);
3853	else
3854	WRITE_ONCE(tp->linger2, val * HZ);
3855	return 0;
3856	case TCP_DEFER_ACCEPT:
3857	/* Translate value in seconds to number of retransmits */
3858	WRITE_ONCE(icsk->icsk_accept_queue.rskq_defer_accept,
3859	secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
3860	TCP_RTO_MAX / HZ));
3861	return 0;
3862	case TCP_RTO_MAX_MS:
3863	if (val < MSEC_PER_SEC || val > TCP_RTO_MAX_SEC * MSEC_PER_SEC)
3864	return -EINVAL;
3865	WRITE_ONCE(inet_csk(sk)->icsk_rto_max, msecs_to_jiffies(val));
3866	return 0;
3867	case TCP_RTO_MIN_US: {
3868	int rto_min = usecs_to_jiffies(u: val);
3869
3870	if (rto_min > TCP_RTO_MIN || rto_min < TCP_TIMEOUT_MIN)
3871	return -EINVAL;
3872	WRITE_ONCE(inet_csk(sk)->icsk_rto_min, rto_min);
3873	return 0;
3874	}
3875	case TCP_DELACK_MAX_US: {
3876	int delack_max = usecs_to_jiffies(u: val);
3877
3878	if (delack_max > TCP_DELACK_MAX || delack_max < TCP_TIMEOUT_MIN)
3879	return -EINVAL;
3880	WRITE_ONCE(inet_csk(sk)->icsk_delack_max, delack_max);
3881	return 0;
3882	}
3883	}
3884
3885	sockopt_lock_sock(sk);
3886
3887	switch (optname) {
3888	case TCP_MAXSEG:
3889	/* Values greater than interface MTU won't take effect. However
3890	* at the point when this call is done we typically don't yet
3891	* know which interface is going to be used
3892	*/
3893	if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) {
3894	err = -EINVAL;
3895	break;
3896	}
3897	tp->rx_opt.user_mss = val;
3898	break;
3899
3900	case TCP_NODELAY:
3901	__tcp_sock_set_nodelay(sk, on: val);
3902	break;
3903
3904	case TCP_THIN_LINEAR_TIMEOUTS:
3905	if (val < 0 || val > 1)
3906	err = -EINVAL;
3907	else
3908	tp->thin_lto = val;
3909	break;
3910
3911	case TCP_THIN_DUPACK:
3912	if (val < 0 || val > 1)
3913	err = -EINVAL;
3914	break;
3915
3916	case TCP_REPAIR:
3917	if (!tcp_can_repair_sock(sk))
3918	err = -EPERM;
3919	else if (val == TCP_REPAIR_ON) {
3920	tp->repair = 1;
3921	sk->sk_reuse = SK_FORCE_REUSE;
3922	tp->repair_queue = TCP_NO_QUEUE;
3923	} else if (val == TCP_REPAIR_OFF) {
3924	tp->repair = 0;
3925	sk->sk_reuse = SK_NO_REUSE;
3926	tcp_send_window_probe(sk);
3927	} else if (val == TCP_REPAIR_OFF_NO_WP) {
3928	tp->repair = 0;
3929	sk->sk_reuse = SK_NO_REUSE;
3930	} else
3931	err = -EINVAL;
3932
3933	break;
3934
3935	case TCP_REPAIR_QUEUE:
3936	if (!tp->repair)
3937	err = -EPERM;
3938	else if ((unsigned int)val < TCP_QUEUES_NR)
3939	tp->repair_queue = val;
3940	else
3941	err = -EINVAL;
3942	break;
3943
3944	case TCP_QUEUE_SEQ:
3945	if (sk->sk_state != TCP_CLOSE) {
3946	err = -EPERM;
3947	} else if (tp->repair_queue == TCP_SEND_QUEUE) {
3948	if (!tcp_rtx_queue_empty(sk))
3949	err = -EPERM;
3950	else
3951	WRITE_ONCE(tp->write_seq, val);
3952	} else if (tp->repair_queue == TCP_RECV_QUEUE) {
3953	if (tp->rcv_nxt != tp->copied_seq) {
3954	err = -EPERM;
3955	} else {
3956	WRITE_ONCE(tp->rcv_nxt, val);
3957	WRITE_ONCE(tp->copied_seq, val);
3958	}
3959	} else {
3960	err = -EINVAL;
3961	}
3962	break;
3963
3964	case TCP_REPAIR_OPTIONS:
3965	if (!tp->repair)
3966	err = -EINVAL;
3967	else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent)
3968	err = tcp_repair_options_est(sk, optbuf: optval, len: optlen);
3969	else
3970	err = -EPERM;
3971	break;
3972
3973	case TCP_CORK:
3974	__tcp_sock_set_cork(sk, on: val);
3975	break;
3976
3977	case TCP_KEEPIDLE:
3978	err = tcp_sock_set_keepidle_locked(sk, val);
3979	break;
3980	case TCP_SAVE_SYN:
3981	/* 0: disable, 1: enable, 2: start from ether_header */
3982	if (val < 0 || val > 2)
3983	err = -EINVAL;
3984	else
3985	tp->save_syn = val;
3986	break;
3987
3988	case TCP_WINDOW_CLAMP:
3989	err = tcp_set_window_clamp(sk, val);
3990	break;
3991
3992	case TCP_QUICKACK:
3993	__tcp_sock_set_quickack(sk, val);
3994	break;
3995
3996	case TCP_AO_REPAIR:
3997	if (!tcp_can_repair_sock(sk)) {
3998	err = -EPERM;
3999	break;
4000	}
4001	err = tcp_ao_set_repair(sk, optval, optlen);
4002	break;
4003	#ifdef CONFIG_TCP_AO
4004	case TCP_AO_ADD_KEY:
4005	case TCP_AO_DEL_KEY:
4006	case TCP_AO_INFO: {
4007	/* If this is the first TCP-AO setsockopt() on the socket,
4008	* sk_state has to be LISTEN or CLOSE. Allow TCP_REPAIR
4009	* in any state.
4010	*/
4011	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
4012	goto ao_parse;
4013	if (rcu_dereference_protected(tcp_sk(sk)->ao_info,
4014	lockdep_sock_is_held(sk)))
4015	goto ao_parse;
4016	if (tp->repair)
4017	goto ao_parse;
4018	err = -EISCONN;
4019	break;
4020	ao_parse:
4021	err = tp->af_specific->ao_parse(sk, optname, optval, optlen);
4022	break;
4023	}
4024	#endif
4025	#ifdef CONFIG_TCP_MD5SIG
4026	case TCP_MD5SIG:
4027	case TCP_MD5SIG_EXT:
4028	err = tp->af_specific->md5_parse(sk, optname, optval, optlen);
4029	break;
4030	#endif
4031	case TCP_FASTOPEN:
4032	if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
4033	TCPF_LISTEN))) {
4034	tcp_fastopen_init_key_once(net);
4035
4036	fastopen_queue_tune(sk, backlog: val);
4037	} else {
4038	err = -EINVAL;
4039	}
4040	break;
4041	case TCP_FASTOPEN_CONNECT:
4042	if (val > 1 || val < 0) {
4043	err = -EINVAL;
4044	} else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) &
4045	TFO_CLIENT_ENABLE) {
4046	if (sk->sk_state == TCP_CLOSE)
4047	tp->fastopen_connect = val;
4048	else
4049	err = -EINVAL;
4050	} else {
4051	err = -EOPNOTSUPP;
4052	}
4053	break;
4054	case TCP_FASTOPEN_NO_COOKIE:
4055	if (val > 1 || val < 0)
4056	err = -EINVAL;
4057	else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
4058	err = -EINVAL;
4059	else
4060	tp->fastopen_no_cookie = val;
4061	break;
4062	case TCP_TIMESTAMP:
4063	if (!tp->repair) {
4064	err = -EPERM;
4065	break;
4066	}
4067	/* val is an opaque field,
4068	* and low order bit contains usec_ts enable bit.
4069	* Its a best effort, and we do not care if user makes an error.
4070	*/
4071	tp->tcp_usec_ts = val & 1;
4072	WRITE_ONCE(tp->tsoffset, val - tcp_clock_ts(tp->tcp_usec_ts));
4073	break;
4074	case TCP_REPAIR_WINDOW:
4075	err = tcp_repair_set_window(tp, optbuf: optval, len: optlen);
4076	break;
4077	case TCP_NOTSENT_LOWAT:
4078	WRITE_ONCE(tp->notsent_lowat, val);
4079	sk->sk_write_space(sk);
4080	break;
4081	case TCP_INQ:
4082	if (val > 1 || val < 0)
4083	err = -EINVAL;
4084	else
4085	tp->recvmsg_inq = val;
4086	break;
4087	case TCP_TX_DELAY:
4088	if (val)
4089	tcp_enable_tx_delay();
4090	WRITE_ONCE(tp->tcp_tx_delay, val);
4091	break;
4092	default:
4093	err = -ENOPROTOOPT;
4094	break;
4095	}
4096
4097	sockopt_release_sock(sk);
4098	return err;
4099	}
4100
4101	int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
4102	unsigned int optlen)
4103	{
4104	const struct inet_connection_sock *icsk = inet_csk(sk);
4105
4106	if (level != SOL_TCP)
4107	/* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
4108	return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname,
4109	optval, optlen);
4110	return do_tcp_setsockopt(sk, level, optname, optval, optlen);
4111	}
4112	EXPORT_IPV6_MOD(tcp_setsockopt);
4113
4114	static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
4115	struct tcp_info *info)
4116	{
4117	u64 stats[__TCP_CHRONO_MAX], total = 0;
4118	enum tcp_chrono i;
4119
4120	for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
4121	stats[i] = tp->chrono_stat[i - 1];
4122	if (i == tp->chrono_type)
4123	stats[i] += tcp_jiffies32 - tp->chrono_start;
4124	stats[i] *= USEC_PER_SEC / HZ;
4125	total += stats[i];
4126	}
4127
4128	info->tcpi_busy_time = total;
4129	info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED];
4130	info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED];
4131	}
4132
4133	/* Return information about state of tcp endpoint in API format. */
4134	void tcp_get_info(struct sock *sk, struct tcp_info *info)
4135	{
4136	const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
4137	const struct inet_connection_sock *icsk = inet_csk(sk);
4138	unsigned long rate;
4139	u32 now;
4140	u64 rate64;
4141	bool slow;
4142
4143	memset(info, 0, sizeof(*info));
4144	if (sk->sk_type != SOCK_STREAM)
4145	return;
4146
4147	info->tcpi_state = inet_sk_state_load(sk);
4148
4149	/* Report meaningful fields for all TCP states, including listeners */
4150	rate = READ_ONCE(sk->sk_pacing_rate);
4151	rate64 = (rate != ~0UL) ? rate : ~0ULL;
4152	info->tcpi_pacing_rate = rate64;
4153
4154	rate = READ_ONCE(sk->sk_max_pacing_rate);
4155	rate64 = (rate != ~0UL) ? rate : ~0ULL;
4156	info->tcpi_max_pacing_rate = rate64;
4157
4158	info->tcpi_reordering = tp->reordering;
4159	info->tcpi_snd_cwnd = tcp_snd_cwnd(tp);
4160
4161	if (info->tcpi_state == TCP_LISTEN) {
4162	/* listeners aliased fields :
4163	* tcpi_unacked -> Number of children ready for accept()
4164	* tcpi_sacked -> max backlog
4165	*/
4166	info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog);
4167	info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog);
4168	return;
4169	}
4170
4171	slow = lock_sock_fast(sk);
4172
4173	info->tcpi_ca_state = icsk->icsk_ca_state;
4174	info->tcpi_retransmits = icsk->icsk_retransmits;
4175	info->tcpi_probes = icsk->icsk_probes_out;
4176	info->tcpi_backoff = icsk->icsk_backoff;
4177
4178	if (tp->rx_opt.tstamp_ok)
4179	info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
4180	if (tcp_is_sack(tp))
4181	info->tcpi_options |= TCPI_OPT_SACK;
4182	if (tp->rx_opt.wscale_ok) {
4183	info->tcpi_options |= TCPI_OPT_WSCALE;
4184	info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
4185	info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
4186	}
4187
4188	if (tcp_ecn_mode_any(tp))
4189	info->tcpi_options |= TCPI_OPT_ECN;
4190	if (tp->ecn_flags & TCP_ECN_SEEN)
4191	info->tcpi_options |= TCPI_OPT_ECN_SEEN;
4192	if (tp->syn_data_acked)
4193	info->tcpi_options |= TCPI_OPT_SYN_DATA;
4194	if (tp->tcp_usec_ts)
4195	info->tcpi_options |= TCPI_OPT_USEC_TS;
4196	if (tp->syn_fastopen_child)
4197	info->tcpi_options |= TCPI_OPT_TFO_CHILD;
4198
4199	info->tcpi_rto = jiffies_to_usecs(j: icsk->icsk_rto);
4200	info->tcpi_ato = jiffies_to_usecs(min_t(u32, icsk->icsk_ack.ato,
4201	tcp_delack_max(sk)));
4202	info->tcpi_snd_mss = tp->mss_cache;
4203	info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
4204
4205	info->tcpi_unacked = tp->packets_out;
4206	info->tcpi_sacked = tp->sacked_out;
4207
4208	info->tcpi_lost = tp->lost_out;
4209	info->tcpi_retrans = tp->retrans_out;
4210
4211	now = tcp_jiffies32;
4212	info->tcpi_last_data_sent = jiffies_to_msecs(j: now - tp->lsndtime);
4213	info->tcpi_last_data_recv = jiffies_to_msecs(j: now - icsk->icsk_ack.lrcvtime);
4214	info->tcpi_last_ack_recv = jiffies_to_msecs(j: now - tp->rcv_tstamp);
4215
4216	info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
4217	info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
4218	info->tcpi_rtt = tp->srtt_us >> 3;
4219	info->tcpi_rttvar = tp->mdev_us >> 2;
4220	info->tcpi_snd_ssthresh = tp->snd_ssthresh;
4221	info->tcpi_advmss = tp->advmss;
4222
4223	info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3;
4224	info->tcpi_rcv_space = tp->rcvq_space.space;
4225
4226	info->tcpi_total_retrans = tp->total_retrans;
4227
4228	info->tcpi_bytes_acked = tp->bytes_acked;
4229	info->tcpi_bytes_received = tp->bytes_received;
4230	info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);
4231	tcp_get_info_chrono_stats(tp, info);
4232
4233	info->tcpi_segs_out = tp->segs_out;
4234
4235	/* segs_in and data_segs_in can be updated from tcp_segs_in() from BH */
4236	info->tcpi_segs_in = READ_ONCE(tp->segs_in);
4237	info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in);
4238
4239	info->tcpi_min_rtt = tcp_min_rtt(tp);
4240	info->tcpi_data_segs_out = tp->data_segs_out;
4241
4242	info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;
4243	rate64 = tcp_compute_delivery_rate(tp);
4244	if (rate64)
4245	info->tcpi_delivery_rate = rate64;
4246	info->tcpi_delivered = tp->delivered;
4247	info->tcpi_delivered_ce = tp->delivered_ce;
4248	info->tcpi_bytes_sent = tp->bytes_sent;
4249	info->tcpi_bytes_retrans = tp->bytes_retrans;
4250	info->tcpi_dsack_dups = tp->dsack_dups;
4251	info->tcpi_reord_seen = tp->reord_seen;
4252	info->tcpi_rcv_ooopack = tp->rcv_ooopack;
4253	info->tcpi_snd_wnd = tp->snd_wnd;
4254	info->tcpi_rcv_wnd = tp->rcv_wnd;
4255	info->tcpi_rehash = tp->plb_rehash + tp->timeout_rehash;
4256	info->tcpi_fastopen_client_fail = tp->fastopen_client_fail;
4257
4258	info->tcpi_total_rto = tp->total_rto;
4259	info->tcpi_total_rto_recoveries = tp->total_rto_recoveries;
4260	info->tcpi_total_rto_time = tp->total_rto_time;
4261	if (tp->rto_stamp)
4262	info->tcpi_total_rto_time += tcp_clock_ms() - tp->rto_stamp;
4263
4264	unlock_sock_fast(sk, slow);
4265	}
4266	EXPORT_SYMBOL_GPL(tcp_get_info);
4267
4268	static size_t tcp_opt_stats_get_size(void)
4269	{
4270	return
4271	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_BUSY */
4272	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */
4273	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */
4274	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */
4275	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */
4276	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_PACING_RATE */
4277	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */
4278	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_SND_CWND */
4279	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_REORDERING */
4280	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_MIN_RTT */
4281	nla_total_size(payload: sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */
4282	nla_total_size(payload: sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */
4283	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */
4284	nla_total_size(payload: sizeof(u8)) + /* TCP_NLA_CA_STATE */
4285	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */
4286	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_DELIVERED */
4287	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */
4288	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_BYTES_SENT */
4289	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */
4290	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */
4291	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_REORD_SEEN */
4292	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_SRTT */
4293	nla_total_size(payload: sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */
4294	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */
4295	nla_total_size_64bit(payload: sizeof(u64)) + /* TCP_NLA_EDT */
4296	nla_total_size(payload: sizeof(u8)) + /* TCP_NLA_TTL */
4297	nla_total_size(payload: sizeof(u32)) + /* TCP_NLA_REHASH */
4298	0;
4299	}
4300
4301	/* Returns TTL or hop limit of an incoming packet from skb. */
4302	static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb)
4303	{
4304	if (skb->protocol == htons(ETH_P_IP))
4305	return ip_hdr(skb)->ttl;
4306	else if (skb->protocol == htons(ETH_P_IPV6))
4307	return ipv6_hdr(skb)->hop_limit;
4308	else
4309	return 0;
4310	}
4311
4312	struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk,
4313	const struct sk_buff *orig_skb,
4314	const struct sk_buff *ack_skb)
4315	{
4316	const struct tcp_sock *tp = tcp_sk(sk);
4317	struct sk_buff *stats;
4318	struct tcp_info info;
4319	unsigned long rate;
4320	u64 rate64;
4321
4322	stats = alloc_skb(size: tcp_opt_stats_get_size(), GFP_ATOMIC);
4323	if (!stats)
4324	return NULL;
4325
4326	tcp_get_info_chrono_stats(tp, info: &info);
4327	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_BUSY,
4328	value: info.tcpi_busy_time, padattr: TCP_NLA_PAD);
4329	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_RWND_LIMITED,
4330	value: info.tcpi_rwnd_limited, padattr: TCP_NLA_PAD);
4331	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_SNDBUF_LIMITED,
4332	value: info.tcpi_sndbuf_limited, padattr: TCP_NLA_PAD);
4333	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_DATA_SEGS_OUT,
4334	value: tp->data_segs_out, padattr: TCP_NLA_PAD);
4335	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_TOTAL_RETRANS,
4336	value: tp->total_retrans, padattr: TCP_NLA_PAD);
4337
4338	rate = READ_ONCE(sk->sk_pacing_rate);
4339	rate64 = (rate != ~0UL) ? rate : ~0ULL;
4340	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_PACING_RATE, value: rate64, padattr: TCP_NLA_PAD);
4341
4342	rate64 = tcp_compute_delivery_rate(tp);
4343	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_DELIVERY_RATE, value: rate64, padattr: TCP_NLA_PAD);
4344
4345	nla_put_u32(skb: stats, attrtype: TCP_NLA_SND_CWND, value: tcp_snd_cwnd(tp));
4346	nla_put_u32(skb: stats, attrtype: TCP_NLA_REORDERING, value: tp->reordering);
4347	nla_put_u32(skb: stats, attrtype: TCP_NLA_MIN_RTT, value: tcp_min_rtt(tp));
4348
4349	nla_put_u8(skb: stats, attrtype: TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits);
4350	nla_put_u8(skb: stats, attrtype: TCP_NLA_DELIVERY_RATE_APP_LMT, value: !!tp->rate_app_limited);
4351	nla_put_u32(skb: stats, attrtype: TCP_NLA_SND_SSTHRESH, value: tp->snd_ssthresh);
4352	nla_put_u32(skb: stats, attrtype: TCP_NLA_DELIVERED, value: tp->delivered);
4353	nla_put_u32(skb: stats, attrtype: TCP_NLA_DELIVERED_CE, value: tp->delivered_ce);
4354
4355	nla_put_u32(skb: stats, attrtype: TCP_NLA_SNDQ_SIZE, value: tp->write_seq - tp->snd_una);
4356	nla_put_u8(skb: stats, attrtype: TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state);
4357
4358	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_BYTES_SENT, value: tp->bytes_sent,
4359	padattr: TCP_NLA_PAD);
4360	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_BYTES_RETRANS, value: tp->bytes_retrans,
4361	padattr: TCP_NLA_PAD);
4362	nla_put_u32(skb: stats, attrtype: TCP_NLA_DSACK_DUPS, value: tp->dsack_dups);
4363	nla_put_u32(skb: stats, attrtype: TCP_NLA_REORD_SEEN, value: tp->reord_seen);
4364	nla_put_u32(skb: stats, attrtype: TCP_NLA_SRTT, value: tp->srtt_us >> 3);
4365	nla_put_u16(skb: stats, attrtype: TCP_NLA_TIMEOUT_REHASH, value: tp->timeout_rehash);
4366	nla_put_u32(skb: stats, attrtype: TCP_NLA_BYTES_NOTSENT,
4367	max_t(int, 0, tp->write_seq - tp->snd_nxt));
4368	nla_put_u64_64bit(skb: stats, attrtype: TCP_NLA_EDT, value: orig_skb->skb_mstamp_ns,
4369	padattr: TCP_NLA_PAD);
4370	if (ack_skb)
4371	nla_put_u8(skb: stats, attrtype: TCP_NLA_TTL,
4372	value: tcp_skb_ttl_or_hop_limit(skb: ack_skb));
4373
4374	nla_put_u32(skb: stats, attrtype: TCP_NLA_REHASH, value: tp->plb_rehash + tp->timeout_rehash);
4375	return stats;
4376	}
4377
4378	int do_tcp_getsockopt(struct sock *sk, int level,
4379	int optname, sockptr_t optval, sockptr_t optlen)
4380	{
4381	struct inet_connection_sock *icsk = inet_csk(sk);
4382	struct tcp_sock *tp = tcp_sk(sk);
4383	struct net *net = sock_net(sk);
4384	int val, len;
4385
4386	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
4387	return -EFAULT;
4388
4389	if (len < 0)
4390	return -EINVAL;
4391
4392	len = min_t(unsigned int, len, sizeof(int));
4393
4394	switch (optname) {
4395	case TCP_MAXSEG:
4396	val = tp->mss_cache;
4397	if (tp->rx_opt.user_mss &&
4398	((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
4399	val = tp->rx_opt.user_mss;
4400	if (tp->repair)
4401	val = tp->rx_opt.mss_clamp;
4402	break;
4403	case TCP_NODELAY:
4404	val = !!(tp->nonagle&TCP_NAGLE_OFF);
4405	break;
4406	case TCP_CORK:
4407	val = !!(tp->nonagle&TCP_NAGLE_CORK);
4408	break;
4409	case TCP_KEEPIDLE:
4410	val = keepalive_time_when(tp) / HZ;
4411	break;
4412	case TCP_KEEPINTVL:
4413	val = keepalive_intvl_when(tp) / HZ;
4414	break;
4415	case TCP_KEEPCNT:
4416	val = keepalive_probes(tp);
4417	break;
4418	case TCP_SYNCNT:
4419	val = READ_ONCE(icsk->icsk_syn_retries) ? :
4420	READ_ONCE(net->ipv4.sysctl_tcp_syn_retries);
4421	break;
4422	case TCP_LINGER2:
4423	val = READ_ONCE(tp->linger2);
4424	if (val >= 0)
4425	val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ;
4426	break;
4427	case TCP_DEFER_ACCEPT:
4428	val = READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept);
4429	val = retrans_to_secs(retrans: val, TCP_TIMEOUT_INIT / HZ,
4430	TCP_RTO_MAX / HZ);
4431	break;
4432	case TCP_WINDOW_CLAMP:
4433	val = READ_ONCE(tp->window_clamp);
4434	break;
4435	case TCP_INFO: {
4436	struct tcp_info info;
4437
4438	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
4439	return -EFAULT;
4440
4441	tcp_get_info(sk, &info);
4442
4443	len = min_t(unsigned int, len, sizeof(info));
4444	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int)))
4445	return -EFAULT;
4446	if (copy_to_sockptr(dst: optval, src: &info, size: len))
4447	return -EFAULT;
4448	return 0;
4449	}
4450	case TCP_CC_INFO: {
4451	const struct tcp_congestion_ops *ca_ops;
4452	union tcp_cc_info info;
4453	size_t sz = 0;
4454	int attr;
4455
4456	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
4457	return -EFAULT;
4458
4459	ca_ops = icsk->icsk_ca_ops;
4460	if (ca_ops && ca_ops->get_info)
4461	sz = ca_ops->get_info(sk, ~0U, &attr, &info);
4462
4463	len = min_t(unsigned int, len, sz);
4464	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int)))
4465	return -EFAULT;
4466	if (copy_to_sockptr(dst: optval, src: &info, size: len))
4467	return -EFAULT;
4468	return 0;
4469	}
4470	case TCP_QUICKACK:
4471	val = !inet_csk_in_pingpong_mode(sk);
4472	break;
4473
4474	case TCP_CONGESTION:
4475	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
4476	return -EFAULT;
4477	len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
4478	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int)))
4479	return -EFAULT;
4480	if (copy_to_sockptr(dst: optval, src: icsk->icsk_ca_ops->name, size: len))
4481	return -EFAULT;
4482	return 0;
4483
4484	case TCP_ULP:
4485	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
4486	return -EFAULT;
4487	len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
4488	if (!icsk->icsk_ulp_ops) {
4489	len = 0;
4490	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int)))
4491	return -EFAULT;
4492	return 0;
4493	}
4494	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int)))
4495	return -EFAULT;
4496	if (copy_to_sockptr(dst: optval, src: icsk->icsk_ulp_ops->name, size: len))
4497	return -EFAULT;
4498	return 0;
4499
4500	case TCP_FASTOPEN_KEY: {
4501	u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
4502	unsigned int key_len;
4503
4504	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
4505	return -EFAULT;
4506
4507	key_len = tcp_fastopen_get_cipher(net, icsk, key) *
4508	TCP_FASTOPEN_KEY_LENGTH;
4509	len = min_t(unsigned int, len, key_len);
4510	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int)))
4511	return -EFAULT;
4512	if (copy_to_sockptr(dst: optval, src: key, size: len))
4513	return -EFAULT;
4514	return 0;
4515	}
4516	case TCP_THIN_LINEAR_TIMEOUTS:
4517	val = tp->thin_lto;
4518	break;
4519
4520	case TCP_THIN_DUPACK:
4521	val = 0;
4522	break;
4523
4524	case TCP_REPAIR:
4525	val = tp->repair;
4526	break;
4527
4528	case TCP_REPAIR_QUEUE:
4529	if (tp->repair)
4530	val = tp->repair_queue;
4531	else
4532	return -EINVAL;
4533	break;
4534
4535	case TCP_REPAIR_WINDOW: {
4536	struct tcp_repair_window opt;
4537
4538	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
4539	return -EFAULT;
4540
4541	if (len != sizeof(opt))
4542	return -EINVAL;
4543
4544	if (!tp->repair)
4545	return -EPERM;
4546
4547	opt.snd_wl1 = tp->snd_wl1;
4548	opt.snd_wnd = tp->snd_wnd;
4549	opt.max_window = tp->max_window;
4550	opt.rcv_wnd = tp->rcv_wnd;
4551	opt.rcv_wup = tp->rcv_wup;
4552
4553	if (copy_to_sockptr(dst: optval, src: &opt, size: len))
4554	return -EFAULT;
4555	return 0;
4556	}
4557	case TCP_QUEUE_SEQ:
4558	if (tp->repair_queue == TCP_SEND_QUEUE)
4559	val = tp->write_seq;
4560	else if (tp->repair_queue == TCP_RECV_QUEUE)
4561	val = tp->rcv_nxt;
4562	else
4563	return -EINVAL;
4564	break;
4565
4566	case TCP_USER_TIMEOUT:
4567	val = READ_ONCE(icsk->icsk_user_timeout);
4568	break;
4569
4570	case TCP_FASTOPEN:
4571	val = READ_ONCE(icsk->icsk_accept_queue.fastopenq.max_qlen);
4572	break;
4573
4574	case TCP_FASTOPEN_CONNECT:
4575	val = tp->fastopen_connect;
4576	break;
4577
4578	case TCP_FASTOPEN_NO_COOKIE:
4579	val = tp->fastopen_no_cookie;
4580	break;
4581
4582	case TCP_TX_DELAY:
4583	val = READ_ONCE(tp->tcp_tx_delay);
4584	break;
4585
4586	case TCP_TIMESTAMP:
4587	val = tcp_clock_ts(usec_ts: tp->tcp_usec_ts) + READ_ONCE(tp->tsoffset);
4588	if (tp->tcp_usec_ts)
4589	val |= 1;
4590	else
4591	val &= ~1;
4592	break;
4593	case TCP_NOTSENT_LOWAT:
4594	val = READ_ONCE(tp->notsent_lowat);
4595	break;
4596	case TCP_INQ:
4597	val = tp->recvmsg_inq;
4598	break;
4599	case TCP_SAVE_SYN:
4600	val = tp->save_syn;
4601	break;
4602	case TCP_SAVED_SYN: {
4603	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
4604	return -EFAULT;
4605
4606	sockopt_lock_sock(sk);
4607	if (tp->saved_syn) {
4608	if (len < tcp_saved_syn_len(saved_syn: tp->saved_syn)) {
4609	len = tcp_saved_syn_len(saved_syn: tp->saved_syn);
4610	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int))) {
4611	sockopt_release_sock(sk);
4612	return -EFAULT;
4613	}
4614	sockopt_release_sock(sk);
4615	return -EINVAL;
4616	}
4617	len = tcp_saved_syn_len(saved_syn: tp->saved_syn);
4618	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int))) {
4619	sockopt_release_sock(sk);
4620	return -EFAULT;
4621	}
4622	if (copy_to_sockptr(dst: optval, src: tp->saved_syn->data, size: len)) {
4623	sockopt_release_sock(sk);
4624	return -EFAULT;
4625	}
4626	tcp_saved_syn_free(tp);
4627	sockopt_release_sock(sk);
4628	} else {
4629	sockopt_release_sock(sk);
4630	len = 0;
4631	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int)))
4632	return -EFAULT;
4633	}
4634	return 0;
4635	}
4636	#ifdef CONFIG_MMU
4637	case TCP_ZEROCOPY_RECEIVE: {
4638	struct scm_timestamping_internal tss;
4639	struct tcp_zerocopy_receive zc = {};
4640	int err;
4641
4642	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
4643	return -EFAULT;
4644	if (len < 0 ||
4645	len < offsetofend(struct tcp_zerocopy_receive, length))
4646	return -EINVAL;
4647	if (unlikely(len > sizeof(zc))) {
4648	err = check_zeroed_sockptr(src: optval, offset: sizeof(zc),
4649	size: len - sizeof(zc));
4650	if (err < 1)
4651	return err == 0 ? -EINVAL : err;
4652	len = sizeof(zc);
4653	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int)))
4654	return -EFAULT;
4655	}
4656	if (copy_from_sockptr(dst: &zc, src: optval, size: len))
4657	return -EFAULT;
4658	if (zc.reserved)
4659	return -EINVAL;
4660	if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS))
4661	return -EINVAL;
4662	sockopt_lock_sock(sk);
4663	err = tcp_zerocopy_receive(sk, zc: &zc, tss: &tss);
4664	err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname,
4665	&zc, &len, err);
4666	sockopt_release_sock(sk);
4667	if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))
4668	goto zerocopy_rcv_cmsg;
4669	switch (len) {
4670	case offsetofend(struct tcp_zerocopy_receive, msg_flags):
4671	goto zerocopy_rcv_cmsg;
4672	case offsetofend(struct tcp_zerocopy_receive, msg_controllen):
4673	case offsetofend(struct tcp_zerocopy_receive, msg_control):
4674	case offsetofend(struct tcp_zerocopy_receive, flags):
4675	case offsetofend(struct tcp_zerocopy_receive, copybuf_len):
4676	case offsetofend(struct tcp_zerocopy_receive, copybuf_address):
4677	case offsetofend(struct tcp_zerocopy_receive, err):
4678	goto zerocopy_rcv_sk_err;
4679	case offsetofend(struct tcp_zerocopy_receive, inq):
4680	goto zerocopy_rcv_inq;
4681	case offsetofend(struct tcp_zerocopy_receive, length):
4682	default:
4683	goto zerocopy_rcv_out;
4684	}
4685	zerocopy_rcv_cmsg:
4686	if (zc.msg_flags & TCP_CMSG_TS)
4687	tcp_zc_finalize_rx_tstamp(sk, zc: &zc, tss: &tss);
4688	else
4689	zc.msg_flags = 0;
4690	zerocopy_rcv_sk_err:
4691	if (!err)
4692	zc.err = sock_error(sk);
4693	zerocopy_rcv_inq:
4694	zc.inq = tcp_inq_hint(sk);
4695	zerocopy_rcv_out:
4696	if (!err && copy_to_sockptr(dst: optval, src: &zc, size: len))
4697	err = -EFAULT;
4698	return err;
4699	}
4700	#endif
4701	case TCP_AO_REPAIR:
4702	if (!tcp_can_repair_sock(sk))
4703	return -EPERM;
4704	return tcp_ao_get_repair(sk, optval, optlen);
4705	case TCP_AO_GET_KEYS:
4706	case TCP_AO_INFO: {
4707	int err;
4708
4709	sockopt_lock_sock(sk);
4710	if (optname == TCP_AO_GET_KEYS)
4711	err = tcp_ao_get_mkts(sk, optval, optlen);
4712	else
4713	err = tcp_ao_get_sock_info(sk, optval, optlen);
4714	sockopt_release_sock(sk);
4715
4716	return err;
4717	}
4718	case TCP_IS_MPTCP:
4719	val = 0;
4720	break;
4721	case TCP_RTO_MAX_MS:
4722	val = jiffies_to_msecs(j: tcp_rto_max(sk));
4723	break;
4724	case TCP_RTO_MIN_US:
4725	val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_rto_min));
4726	break;
4727	case TCP_DELACK_MAX_US:
4728	val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_delack_max));
4729	break;
4730	default:
4731	return -ENOPROTOOPT;
4732	}
4733
4734	if (copy_to_sockptr(dst: optlen, src: &len, size: sizeof(int)))
4735	return -EFAULT;
4736	if (copy_to_sockptr(dst: optval, src: &val, size: len))
4737	return -EFAULT;
4738	return 0;
4739	}
4740
4741	bool tcp_bpf_bypass_getsockopt(int level, int optname)
4742	{
4743	/* TCP do_tcp_getsockopt has optimized getsockopt implementation
4744	* to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE.
4745	*/
4746	if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE)
4747	return true;
4748
4749	return false;
4750	}
4751	EXPORT_IPV6_MOD(tcp_bpf_bypass_getsockopt);
4752
4753	int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
4754	int __user *optlen)
4755	{
4756	struct inet_connection_sock *icsk = inet_csk(sk);
4757
4758	if (level != SOL_TCP)
4759	/* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
4760	return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname,
4761	optval, optlen);
4762	return do_tcp_getsockopt(sk, level, optname, optval: USER_SOCKPTR(p: optval),
4763	optlen: USER_SOCKPTR(p: optlen));
4764	}
4765	EXPORT_IPV6_MOD(tcp_getsockopt);
4766
4767	#ifdef CONFIG_TCP_MD5SIG
4768	int tcp_md5_sigpool_id = -1;
4769	EXPORT_IPV6_MOD_GPL(tcp_md5_sigpool_id);
4770
4771	int tcp_md5_alloc_sigpool(void)
4772	{
4773	size_t scratch_size;
4774	int ret;
4775
4776	scratch_size = sizeof(union tcp_md5sum_block) + sizeof(struct tcphdr);
4777	ret = tcp_sigpool_alloc_ahash(alg: "md5", scratch_size);
4778	if (ret >= 0) {
4779	/* As long as any md5 sigpool was allocated, the return
4780	* id would stay the same. Re-write the id only for the case
4781	* when previously all MD5 keys were deleted and this call
4782	* allocates the first MD5 key, which may return a different
4783	* sigpool id than was used previously.
4784	*/
4785	WRITE_ONCE(tcp_md5_sigpool_id, ret); /* Avoids the compiler potentially being smart here */
4786	return 0;
4787	}
4788	return ret;
4789	}
4790
4791	void tcp_md5_release_sigpool(void)
4792	{
4793	tcp_sigpool_release(READ_ONCE(tcp_md5_sigpool_id));
4794	}
4795
4796	void tcp_md5_add_sigpool(void)
4797	{
4798	tcp_sigpool_get(READ_ONCE(tcp_md5_sigpool_id));
4799	}
4800
4801	int tcp_md5_hash_key(struct tcp_sigpool *hp,
4802	const struct tcp_md5sig_key *key)
4803	{
4804	u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */
4805	struct scatterlist sg;
4806
4807	sg_init_one(&sg, key->key, keylen);
4808	ahash_request_set_crypt(req: hp->req, src: &sg, NULL, nbytes: keylen);
4809
4810	/* We use data_race() because tcp_md5_do_add() might change
4811	* key->key under us
4812	*/
4813	return data_race(crypto_ahash_update(hp->req));
4814	}
4815	EXPORT_IPV6_MOD(tcp_md5_hash_key);
4816
4817	/* Called with rcu_read_lock() */
4818	static enum skb_drop_reason
4819	tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
4820	const void *saddr, const void *daddr,
4821	int family, int l3index, const __u8 *hash_location)
4822	{
4823	/* This gets called for each TCP segment that has TCP-MD5 option.
4824	* We have 3 drop cases:
4825	* o No MD5 hash and one expected.
4826	* o MD5 hash and we're not expecting one.
4827	* o MD5 hash and its wrong.
4828	*/
4829	const struct tcp_sock *tp = tcp_sk(sk);
4830	struct tcp_md5sig_key *key;
4831	u8 newhash[16];
4832	int genhash;
4833
4834	key = tcp_md5_do_lookup(sk, l3index, addr: saddr, family);
4835
4836	if (!key && hash_location) {
4837	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
4838	trace_tcp_hash_md5_unexpected(sk, skb);
4839	return SKB_DROP_REASON_TCP_MD5UNEXPECTED;
4840	}
4841
4842	/* Check the signature.
4843	* To support dual stack listeners, we need to handle
4844	* IPv4-mapped case.
4845	*/
4846	if (family == AF_INET)
4847	genhash = tcp_v4_md5_hash_skb(md5_hash: newhash, key, NULL, skb);
4848	else
4849	genhash = tp->af_specific->calc_md5_hash(newhash, key,
4850	NULL, skb);
4851	if (genhash || memcmp(p: hash_location, q: newhash, size: 16) != 0) {
4852	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
4853	trace_tcp_hash_md5_mismatch(sk, skb);
4854	return SKB_DROP_REASON_TCP_MD5FAILURE;
4855	}
4856	return SKB_NOT_DROPPED_YET;
4857	}
4858	#else
4859	static inline enum skb_drop_reason
4860	tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
4861	const void *saddr, const void *daddr,
4862	int family, int l3index, const __u8 *hash_location)
4863	{
4864	return SKB_NOT_DROPPED_YET;
4865	}
4866
4867	#endif
4868
4869	/* Called with rcu_read_lock() */
4870	enum skb_drop_reason
4871	tcp_inbound_hash(struct sock *sk, const struct request_sock *req,
4872	const struct sk_buff *skb,
4873	const void *saddr, const void *daddr,
4874	int family, int dif, int sdif)
4875	{
4876	const struct tcphdr *th = tcp_hdr(skb);
4877	const struct tcp_ao_hdr *aoh;
4878	const __u8 *md5_location;
4879	int l3index;
4880
4881	/* Invalid option or two times meet any of auth options */
4882	if (tcp_parse_auth_options(th, md5_hash: &md5_location, aoh: &aoh)) {
4883	trace_tcp_hash_bad_header(sk, skb);
4884	return SKB_DROP_REASON_TCP_AUTH_HDR;
4885	}
4886
4887	if (req) {
4888	if (tcp_rsk_used_ao(req) != !!aoh) {
4889	u8 keyid, rnext, maclen;
4890
4891	if (aoh) {
4892	keyid = aoh->keyid;
4893	rnext = aoh->rnext_keyid;
4894	maclen = tcp_ao_hdr_maclen(aoh);
4895	} else {
4896	keyid = rnext = maclen = 0;
4897	}
4898
4899	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
4900	trace_tcp_ao_handshake_failure(sk, skb, keyid, rnext, maclen);
4901	return SKB_DROP_REASON_TCP_AOFAILURE;
4902	}
4903	}
4904
4905	/* sdif set, means packet ingressed via a device
4906	* in an L3 domain and dif is set to the l3mdev
4907	*/
4908	l3index = sdif ? dif : 0;
4909
4910	/* Fast path: unsigned segments */
4911	if (likely(!md5_location && !aoh)) {
4912	/* Drop if there's TCP-MD5 or TCP-AO key with any rcvid/sndid
4913	* for the remote peer. On TCP-AO established connection
4914	* the last key is impossible to remove, so there's
4915	* always at least one current_key.
4916	*/
4917	if (tcp_ao_required(sk, saddr, family, l3index, stat_inc: true)) {
4918	trace_tcp_hash_ao_required(sk, skb);
4919	return SKB_DROP_REASON_TCP_AONOTFOUND;
4920	}
4921	if (unlikely(tcp_md5_do_lookup(sk, l3index, saddr, family))) {
4922	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
4923	trace_tcp_hash_md5_required(sk, skb);
4924	return SKB_DROP_REASON_TCP_MD5NOTFOUND;
4925	}
4926	return SKB_NOT_DROPPED_YET;
4927	}
4928
4929	if (aoh)
4930	return tcp_inbound_ao_hash(sk, skb, family, req, l3index, aoh);
4931
4932	return tcp_inbound_md5_hash(sk, skb, saddr, daddr, family,
4933	l3index, hash_location: md5_location);
4934	}
4935	EXPORT_IPV6_MOD_GPL(tcp_inbound_hash);
4936
4937	void tcp_done(struct sock *sk)
4938	{
4939	struct request_sock *req;
4940
4941	/* We might be called with a new socket, after
4942	* inet_csk_prepare_forced_close() has been called
4943	* so we can not use lockdep_sock_is_held(sk)
4944	*/
4945	req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1);
4946
4947	if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
4948	TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
4949
4950	tcp_set_state(sk, TCP_CLOSE);
4951	tcp_clear_xmit_timers(sk);
4952	if (req)
4953	reqsk_fastopen_remove(sk, req, reset: false);
4954
4955	WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
4956
4957	if (!sock_flag(sk, flag: SOCK_DEAD))
4958	sk->sk_state_change(sk);
4959	else
4960	inet_csk_destroy_sock(sk);
4961	}
4962	EXPORT_SYMBOL_GPL(tcp_done);
4963
4964	int tcp_abort(struct sock *sk, int err)
4965	{
4966	int state = inet_sk_state_load(sk);
4967
4968	if (state == TCP_NEW_SYN_RECV) {
4969	struct request_sock *req = inet_reqsk(sk);
4970
4971	local_bh_disable();
4972	inet_csk_reqsk_queue_drop(sk: req->rsk_listener, req);
4973	local_bh_enable();
4974	return 0;
4975	}
4976	if (state == TCP_TIME_WAIT) {
4977	struct inet_timewait_sock *tw = inet_twsk(sk);
4978
4979	refcount_inc(r: &tw->tw_refcnt);
4980	local_bh_disable();
4981	inet_twsk_deschedule_put(tw);
4982	local_bh_enable();
4983	return 0;
4984	}
4985
4986	/* BPF context ensures sock locking. */
4987	if (!has_current_bpf_ctx())
4988	/* Don't race with userspace socket closes such as tcp_close. */
4989	lock_sock(sk);
4990
4991	/* Avoid closing the same socket twice. */
4992	if (sk->sk_state == TCP_CLOSE) {
4993	if (!has_current_bpf_ctx())
4994	release_sock(sk);
4995	return -ENOENT;
4996	}
4997
4998	if (sk->sk_state == TCP_LISTEN) {
4999	tcp_set_state(sk, TCP_CLOSE);
5000	inet_csk_listen_stop(sk);
5001	}
5002
5003	/* Don't race with BH socket closes such as inet_csk_listen_stop. */
5004	local_bh_disable();
5005	bh_lock_sock(sk);
5006
5007	if (tcp_need_reset(state: sk->sk_state))
5008	tcp_send_active_reset(sk, GFP_ATOMIC,
5009	reason: SK_RST_REASON_TCP_STATE);
5010	tcp_done_with_error(sk, err);
5011
5012	bh_unlock_sock(sk);
5013	local_bh_enable();
5014	if (!has_current_bpf_ctx())
5015	release_sock(sk);
5016	return 0;
5017	}
5018	EXPORT_SYMBOL_GPL(tcp_abort);
5019
5020	extern struct tcp_congestion_ops tcp_reno;
5021
5022	static __initdata unsigned long thash_entries;
5023	static int __init set_thash_entries(char *str)
5024	{
5025	ssize_t ret;
5026
5027	if (!str)
5028	return 0;
5029
5030	ret = kstrtoul(s: str, base: 0, res: &thash_entries);
5031	if (ret)
5032	return 0;
5033
5034	return 1;
5035	}
5036	__setup("thash_entries=", set_thash_entries);
5037
5038	static void __init tcp_init_mem(void)
5039	{
5040	unsigned long limit = nr_free_buffer_pages() / 16;
5041
5042	limit = max(limit, 128UL);
5043	sysctl_tcp_mem[0] = limit / 4 * 3; /* 4.68 % */
5044	sysctl_tcp_mem[1] = limit; /* 6.25 % */
5045	sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */
5046	}
5047
5048	static void __init tcp_struct_check(void)
5049	{
5050	/* TX read-mostly hotpath cache lines */
5051	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, max_window);
5052	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, rcv_ssthresh);
5053	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, reordering);
5054	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, notsent_lowat);
5055	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, gso_segs);
5056	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, lost_skb_hint);
5057	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, retransmit_skb_hint);
5058	CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_tx, 40);
5059
5060	/* TXRX read-mostly hotpath cache lines */
5061	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, tsoffset);
5062	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_wnd);
5063	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, mss_cache);
5064	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_cwnd);
5065	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, prr_out);
5066	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out);
5067	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out);
5068	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio);
5069	CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_txrx, 32);
5070
5071	/* RX read-mostly hotpath cache lines */
5072	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq);
5073	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rcv_tstamp);
5074	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_wl1);
5075	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tlp_high_seq);
5076	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rttvar_us);
5077	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, retrans_out);
5078	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, advmss);
5079	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, urg_data);
5080	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, lost);
5081	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rtt_min);
5082	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, out_of_order_queue);
5083	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh);
5084	#if IS_ENABLED(CONFIG_TLS_DEVICE)
5085	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tcp_clean_acked);
5086	CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_rx, 77);
5087	#else
5088	CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_rx, 69);
5089	#endif
5090
5091	/* TX read-write hotpath cache lines */
5092	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, segs_out);
5093	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out);
5094	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent);
5095	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml);
5096	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start);
5097	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat);
5098	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, write_seq);
5099	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, pushed_seq);
5100	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, lsndtime);
5101	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, mdev_us);
5102	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns);
5103	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq);
5104	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue);
5105	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack);
5106	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags);
5107	CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_tx, 89);
5108
5109	/* TXRX read-write hotpath cache lines */
5110	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, pred_flags);
5111	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_clock_cache);
5112	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_mstamp);
5113	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_nxt);
5114	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_nxt);
5115	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_una);
5116	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, window_clamp);
5117	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us);
5118	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out);
5119	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up);
5120	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered);
5121	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered_ce);
5122	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited);
5123	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_wnd);
5124	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt);
5125
5126	/* 32bit arches with 8byte alignment on u64 fields might need padding
5127	* before tcp_clock_cache.
5128	*/
5129	CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_txrx, 92 + 4);
5130
5131	/* RX read-write hotpath cache lines */
5132	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received);
5133	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, segs_in);
5134	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in);
5135	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup);
5136	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, max_packets_out);
5137	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, cwnd_usage_seq);
5138	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_delivered);
5139	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_interval_us);
5140	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr);
5141	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, first_tx_mstamp);
5142	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_mstamp);
5143	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_acked);
5144	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est);
5145	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space);
5146	CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_rx, 99);
5147	}
5148
5149	void __init tcp_init(void)
5150	{
5151	int max_rshare, max_wshare, cnt;
5152	unsigned long limit;
5153	unsigned int i;
5154
5155	BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
5156	BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
5157	sizeof_field(struct sk_buff, cb));
5158
5159	tcp_struct_check();
5160
5161	percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
5162
5163	timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE);
5164	mod_timer(timer: &tcp_orphan_timer, expires: jiffies + TCP_ORPHAN_TIMER_PERIOD);
5165
5166	inet_hashinfo2_init(h: &tcp_hashinfo, name: "tcp_listen_portaddr_hash",
5167	numentries: thash_entries, scale: 21, /* one slot per 2 MB*/
5168	low_limit: 0, high_limit: 64 * 1024);
5169	tcp_hashinfo.bind_bucket_cachep =
5170	kmem_cache_create("tcp_bind_bucket",
5171	sizeof(struct inet_bind_bucket), 0,
5172	SLAB_HWCACHE_ALIGN | SLAB_PANIC |
5173	SLAB_ACCOUNT,
5174	NULL);
5175	tcp_hashinfo.bind2_bucket_cachep =
5176	kmem_cache_create("tcp_bind2_bucket",
5177	sizeof(struct inet_bind2_bucket), 0,
5178	SLAB_HWCACHE_ALIGN | SLAB_PANIC |
5179	SLAB_ACCOUNT,
5180	NULL);
5181
5182	/* Size and allocate the main established and bind bucket
5183	* hash tables.
5184	*
5185	* The methodology is similar to that of the buffer cache.
5186	*/
5187	tcp_hashinfo.ehash =
5188	alloc_large_system_hash(tablename: "TCP established",
5189	bucketsize: sizeof(struct inet_ehash_bucket),
5190	numentries: thash_entries,
5191	scale: 17, /* one slot per 128 KB of memory */
5192	flags: 0,
5193	NULL,
5194	hash_mask: &tcp_hashinfo.ehash_mask,
5195	low_limit: 0,
5196	high_limit: thash_entries ? 0 : 512 * 1024);
5197	for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
5198	INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
5199
5200	if (inet_ehash_locks_alloc(hashinfo: &tcp_hashinfo))
5201	panic(fmt: "TCP: failed to alloc ehash_locks");
5202	tcp_hashinfo.bhash =
5203	alloc_large_system_hash(tablename: "TCP bind",
5204	bucketsize: 2 * sizeof(struct inet_bind_hashbucket),
5205	numentries: tcp_hashinfo.ehash_mask + 1,
5206	scale: 17, /* one slot per 128 KB of memory */
5207	flags: 0,
5208	hash_shift: &tcp_hashinfo.bhash_size,
5209	NULL,
5210	low_limit: 0,
5211	high_limit: 64 * 1024);
5212	tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
5213	tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;
5214	for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
5215	spin_lock_init(&tcp_hashinfo.bhash[i].lock);
5216	INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
5217	spin_lock_init(&tcp_hashinfo.bhash2[i].lock);
5218	INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain);
5219	}
5220
5221	tcp_hashinfo.pernet = false;
5222
5223	cnt = tcp_hashinfo.ehash_mask + 1;
5224	sysctl_tcp_max_orphans = cnt / 2;
5225
5226	tcp_init_mem();
5227	/* Set per-socket limits to no more than 1/128 the pressure threshold */
5228	limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
5229	max_wshare = min(4UL*1024*1024, limit);
5230	max_rshare = min(32UL*1024*1024, limit);
5231
5232	init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE;
5233	init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
5234	init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
5235
5236	init_net.ipv4.sysctl_tcp_rmem[0] = PAGE_SIZE;
5237	init_net.ipv4.sysctl_tcp_rmem[1] = 131072;
5238	init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare);
5239
5240	pr_info("Hash tables configured (established %u bind %u)\n",
5241	tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
5242
5243	tcp_v4_init();
5244	tcp_metrics_init();
5245	BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
5246	tcp_tasklet_init();
5247	mptcp_init();
5248	}
5249