1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* TCP over IPv6
4	* Linux INET6 implementation
5	*
6	* Authors:
7	* Pedro Roque <roque@di.fc.ul.pt>
8	*
9	* Based on:
10	* linux/net/ipv4/tcp.c
11	* linux/net/ipv4/tcp_input.c
12	* linux/net/ipv4/tcp_output.c
13	*
14	* Fixes:
15	* Hideaki YOSHIFUJI : sin6_scope_id support
16	* YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
17	* Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
18	* a single port at the same time.
19	* YOSHIFUJI Hideaki @USAGI: convert /proc/net/tcp6 to seq_file.
20	*/
21
22	#include <linux/bottom_half.h>
23	#include <linux/module.h>
24	#include <linux/errno.h>
25	#include <linux/types.h>
26	#include <linux/socket.h>
27	#include <linux/sockios.h>
28	#include <linux/net.h>
29	#include <linux/jiffies.h>
30	#include <linux/in.h>
31	#include <linux/in6.h>
32	#include <linux/netdevice.h>
33	#include <linux/init.h>
34	#include <linux/jhash.h>
35	#include <linux/ipsec.h>
36	#include <linux/times.h>
37	#include <linux/slab.h>
38	#include <linux/uaccess.h>
39	#include <linux/ipv6.h>
40	#include <linux/icmpv6.h>
41	#include <linux/random.h>
42	#include <linux/indirect_call_wrapper.h>
43
44	#include <net/tcp.h>
45	#include <net/ndisc.h>
46	#include <net/inet6_hashtables.h>
47	#include <net/inet6_connection_sock.h>
48	#include <net/ipv6.h>
49	#include <net/transp_v6.h>
50	#include <net/addrconf.h>
51	#include <net/ip6_route.h>
52	#include <net/ip6_checksum.h>
53	#include <net/inet_ecn.h>
54	#include <net/protocol.h>
55	#include <net/xfrm.h>
56	#include <net/snmp.h>
57	#include <net/dsfield.h>
58	#include <net/timewait_sock.h>
59	#include <net/inet_common.h>
60	#include <net/secure_seq.h>
61	#include <net/hotdata.h>
62	#include <net/busy_poll.h>
63	#include <net/rstreason.h>
64
65	#include <linux/proc_fs.h>
66	#include <linux/seq_file.h>
67
68	#include <crypto/hash.h>
69	#include <linux/scatterlist.h>
70
71	#include <trace/events/tcp.h>
72
73	static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
74	enum sk_rst_reason reason);
75	static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
76	struct request_sock *req);
77
78	INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
79
80	static const struct inet_connection_sock_af_ops ipv6_mapped;
81	const struct inet_connection_sock_af_ops ipv6_specific;
82	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
83	static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
84	static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
85	#endif
86
87	/* Helper returning the inet6 address from a given tcp socket.
88	* It can be used in TCP stack instead of inet6_sk(sk).
89	* This avoids a dereference and allow compiler optimizations.
90	* It is a specialized version of inet6_sk_generic().
91	*/
92	#define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
93	struct tcp6_sock, tcp)->inet6)
94
95	static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
96	{
97	struct dst_entry *dst = skb_dst(skb);
98
99	if (dst && dst_hold_safe(dst)) {
100	rcu_assign_pointer(sk->sk_rx_dst, dst);
101	sk->sk_rx_dst_ifindex = skb->skb_iif;
102	sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst));
103	}
104	}
105
106	static u32 tcp_v6_init_seq(const struct sk_buff *skb)
107	{
108	return secure_tcpv6_seq(saddr: ipv6_hdr(skb)->daddr.s6_addr32,
109	daddr: ipv6_hdr(skb)->saddr.s6_addr32,
110	sport: tcp_hdr(skb)->dest,
111	dport: tcp_hdr(skb)->source);
112	}
113
114	static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb)
115	{
116	return secure_tcpv6_ts_off(net, saddr: ipv6_hdr(skb)->daddr.s6_addr32,
117	daddr: ipv6_hdr(skb)->saddr.s6_addr32);
118	}
119
120	static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
121	int addr_len)
122	{
123	/* This check is replicated from tcp_v6_connect() and intended to
124	* prevent BPF program called below from accessing bytes that are out
125	* of the bound specified by user in addr_len.
126	*/
127	if (addr_len < SIN6_LEN_RFC2133)
128	return -EINVAL;
129
130	sock_owned_by_me(sk);
131
132	return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
133	}
134
135	static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
136	int addr_len)
137	{
138	struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
139	struct inet_connection_sock *icsk = inet_csk(sk);
140	struct in6_addr *saddr = NULL, *final_p, final;
141	struct inet_timewait_death_row *tcp_death_row;
142	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
143	struct inet_sock *inet = inet_sk(sk);
144	struct tcp_sock *tp = tcp_sk(sk);
145	struct net *net = sock_net(sk);
146	struct ipv6_txoptions *opt;
147	struct dst_entry *dst;
148	struct flowi6 fl6;
149	int addr_type;
150	int err;
151
152	if (addr_len < SIN6_LEN_RFC2133)
153	return -EINVAL;
154
155	if (usin->sin6_family != AF_INET6)
156	return -EAFNOSUPPORT;
157
158	memset(&fl6, 0, sizeof(fl6));
159
160	if (inet6_test_bit(SNDFLOW, sk)) {
161	fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
162	IP6_ECN_flow_init(fl6.flowlabel);
163	if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
164	struct ip6_flowlabel *flowlabel;
165	flowlabel = fl6_sock_lookup(sk, label: fl6.flowlabel);
166	if (IS_ERR(ptr: flowlabel))
167	return -EINVAL;
168	fl6_sock_release(fl: flowlabel);
169	}
170	}
171
172	/*
173	* connect() to INADDR_ANY means loopback (BSD'ism).
174	*/
175
176	if (ipv6_addr_any(a: &usin->sin6_addr)) {
177	if (ipv6_addr_v4mapped(a: &sk->sk_v6_rcv_saddr))
178	ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
179	v4mapped: &usin->sin6_addr);
180	else
181	usin->sin6_addr = in6addr_loopback;
182	}
183
184	addr_type = ipv6_addr_type(addr: &usin->sin6_addr);
185
186	if (addr_type & IPV6_ADDR_MULTICAST)
187	return -ENETUNREACH;
188
189	if (addr_type&IPV6_ADDR_LINKLOCAL) {
190	if (addr_len >= sizeof(struct sockaddr_in6) &&
191	usin->sin6_scope_id) {
192	/* If interface is set while binding, indices
193	* must coincide.
194	*/
195	if (!sk_dev_equal_l3scope(sk, dif: usin->sin6_scope_id))
196	return -EINVAL;
197
198	sk->sk_bound_dev_if = usin->sin6_scope_id;
199	}
200
201	/* Connect to link-local address requires an interface */
202	if (!sk->sk_bound_dev_if)
203	return -EINVAL;
204	}
205
206	if (tp->rx_opt.ts_recent_stamp &&
207	!ipv6_addr_equal(a1: &sk->sk_v6_daddr, a2: &usin->sin6_addr)) {
208	tp->rx_opt.ts_recent = 0;
209	tp->rx_opt.ts_recent_stamp = 0;
210	WRITE_ONCE(tp->write_seq, 0);
211	}
212
213	sk->sk_v6_daddr = usin->sin6_addr;
214	np->flow_label = fl6.flowlabel;
215
216	/*
217	* TCP over IPv4
218	*/
219
220	if (addr_type & IPV6_ADDR_MAPPED) {
221	u32 exthdrlen = icsk->icsk_ext_hdr_len;
222	struct sockaddr_in sin;
223
224	if (ipv6_only_sock(sk))
225	return -ENETUNREACH;
226
227	sin.sin_family = AF_INET;
228	sin.sin_port = usin->sin6_port;
229	sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
230
231	/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
232	WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
233	if (sk_is_mptcp(sk))
234	mptcpv6_handle_mapped(sk, mapped: true);
235	sk->sk_backlog_rcv = tcp_v4_do_rcv;
236	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
237	tp->af_specific = &tcp_sock_ipv6_mapped_specific;
238	#endif
239
240	err = tcp_v4_connect(sk, uaddr: (struct sockaddr *)&sin, addr_len: sizeof(sin));
241
242	if (err) {
243	icsk->icsk_ext_hdr_len = exthdrlen;
244	/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
245	WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
246	if (sk_is_mptcp(sk))
247	mptcpv6_handle_mapped(sk, mapped: false);
248	sk->sk_backlog_rcv = tcp_v6_do_rcv;
249	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
250	tp->af_specific = &tcp_sock_ipv6_specific;
251	#endif
252	goto failure;
253	}
254	np->saddr = sk->sk_v6_rcv_saddr;
255
256	return err;
257	}
258
259	if (!ipv6_addr_any(a: &sk->sk_v6_rcv_saddr))
260	saddr = &sk->sk_v6_rcv_saddr;
261
262	fl6.flowi6_proto = IPPROTO_TCP;
263	fl6.daddr = sk->sk_v6_daddr;
264	fl6.saddr = saddr ? *saddr : np->saddr;
265	fl6.flowlabel = ip6_make_flowinfo(tclass: np->tclass, flowlabel: np->flow_label);
266	fl6.flowi6_oif = sk->sk_bound_dev_if;
267	fl6.flowi6_mark = sk->sk_mark;
268	fl6.fl6_dport = usin->sin6_port;
269	fl6.fl6_sport = inet->inet_sport;
270	if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6.fl6_sport)
271	fl6.flowi6_flags = FLOWI_FLAG_ANY_SPORT;
272	fl6.flowi6_uid = sk->sk_uid;
273
274	opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
275	final_p = fl6_update_dst(fl6: &fl6, opt, orig: &final);
276
277	security_sk_classify_flow(sk, flic: flowi6_to_flowi_common(fl6: &fl6));
278
279	dst = ip6_dst_lookup_flow(net, sk, fl6: &fl6, final_dst: final_p);
280	if (IS_ERR(ptr: dst)) {
281	err = PTR_ERR(ptr: dst);
282	goto failure;
283	}
284
285	tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
286	tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
287
288	if (!saddr) {
289	saddr = &fl6.saddr;
290
291	err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
292	if (err)
293	goto failure;
294	}
295
296	/* set the source address */
297	np->saddr = *saddr;
298	inet->inet_rcv_saddr = LOOPBACK4_IPV6;
299
300	sk->sk_gso_type = SKB_GSO_TCPV6;
301	ip6_dst_store(sk, dst, NULL, NULL);
302
303	icsk->icsk_ext_hdr_len = 0;
304	if (opt)
305	icsk->icsk_ext_hdr_len = opt->opt_flen +
306	opt->opt_nflen;
307
308	tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
309
310	inet->inet_dport = usin->sin6_port;
311
312	tcp_set_state(sk, state: TCP_SYN_SENT);
313	err = inet6_hash_connect(death_row: tcp_death_row, sk);
314	if (err)
315	goto late_failure;
316
317	sk_set_txhash(sk);
318
319	if (likely(!tp->repair)) {
320	if (!tp->write_seq)
321	WRITE_ONCE(tp->write_seq,
322	secure_tcpv6_seq(np->saddr.s6_addr32,
323	sk->sk_v6_daddr.s6_addr32,
324	inet->inet_sport,
325	inet->inet_dport));
326	tp->tsoffset = secure_tcpv6_ts_off(net, saddr: np->saddr.s6_addr32,
327	daddr: sk->sk_v6_daddr.s6_addr32);
328	}
329
330	if (tcp_fastopen_defer_connect(sk, err: &err))
331	return err;
332	if (err)
333	goto late_failure;
334
335	err = tcp_connect(sk);
336	if (err)
337	goto late_failure;
338
339	return 0;
340
341	late_failure:
342	tcp_set_state(sk, state: TCP_CLOSE);
343	inet_bhash2_reset_saddr(sk);
344	failure:
345	inet->inet_dport = 0;
346	sk->sk_route_caps = 0;
347	return err;
348	}
349
350	static void tcp_v6_mtu_reduced(struct sock *sk)
351	{
352	struct dst_entry *dst;
353	u32 mtu;
354
355	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
356	return;
357
358	mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
359
360	/* Drop requests trying to increase our current mss.
361	* Check done in __ip6_rt_update_pmtu() is too late.
362	*/
363	if (tcp_mtu_to_mss(sk, pmtu: mtu) >= tcp_sk(sk)->mss_cache)
364	return;
365
366	dst = inet6_csk_update_pmtu(sk, mtu);
367	if (!dst)
368	return;
369
370	if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
371	tcp_sync_mss(sk, pmtu: dst_mtu(dst));
372	tcp_simple_retransmit(sk);
373	}
374	}
375
376	static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
377	u8 type, u8 code, int offset, __be32 info)
378	{
379	const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
380	const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
381	struct net *net = dev_net_rcu(dev: skb->dev);
382	struct request_sock *fastopen;
383	struct ipv6_pinfo *np;
384	struct tcp_sock *tp;
385	__u32 seq, snd_una;
386	struct sock *sk;
387	bool fatal;
388	int err;
389
390	sk = __inet6_lookup_established(net, hashinfo: net->ipv4.tcp_death_row.hashinfo,
391	saddr: &hdr->daddr, sport: th->dest,
392	daddr: &hdr->saddr, ntohs(th->source),
393	dif: skb->dev->ifindex, sdif: inet6_sdif(skb));
394
395	if (!sk) {
396	__ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
397	ICMP6_MIB_INERRORS);
398	return -ENOENT;
399	}
400
401	if (sk->sk_state == TCP_TIME_WAIT) {
402	/* To increase the counter of ignored icmps for TCP-AO */
403	tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
404	inet_twsk_put(tw: inet_twsk(sk));
405	return 0;
406	}
407	seq = ntohl(th->seq);
408	fatal = icmpv6_err_convert(type, code, err: &err);
409	if (sk->sk_state == TCP_NEW_SYN_RECV) {
410	tcp_req_err(sk, seq, abort: fatal);
411	return 0;
412	}
413
414	if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
415	sock_put(sk);
416	return 0;
417	}
418
419	bh_lock_sock(sk);
420	if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
421	__NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
422
423	if (sk->sk_state == TCP_CLOSE)
424	goto out;
425
426	if (static_branch_unlikely(&ip6_min_hopcount)) {
427	/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
428	if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
429	__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
430	goto out;
431	}
432	}
433
434	tp = tcp_sk(sk);
435	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
436	fastopen = rcu_dereference(tp->fastopen_rsk);
437	snd_una = fastopen ? tcp_rsk(req: fastopen)->snt_isn : tp->snd_una;
438	if (sk->sk_state != TCP_LISTEN &&
439	!between(seq1: seq, seq2: snd_una, seq3: tp->snd_nxt)) {
440	__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
441	goto out;
442	}
443
444	np = tcp_inet6_sk(sk);
445
446	if (type == NDISC_REDIRECT) {
447	if (!sock_owned_by_user(sk)) {
448	struct dst_entry *dst = __sk_dst_check(sk, cookie: np->dst_cookie);
449
450	if (dst)
451	dst->ops->redirect(dst, sk, skb);
452	}
453	goto out;
454	}
455
456	if (type == ICMPV6_PKT_TOOBIG) {
457	u32 mtu = ntohl(info);
458
459	/* We are not interested in TCP_LISTEN and open_requests
460	* (SYN-ACKs send out by Linux are always <576bytes so
461	* they should go through unfragmented).
462	*/
463	if (sk->sk_state == TCP_LISTEN)
464	goto out;
465
466	if (!ip6_sk_accept_pmtu(sk))
467	goto out;
468
469	if (mtu < IPV6_MIN_MTU)
470	goto out;
471
472	WRITE_ONCE(tp->mtu_info, mtu);
473
474	if (!sock_owned_by_user(sk))
475	tcp_v6_mtu_reduced(sk);
476	else if (!test_and_set_bit(nr: TCP_MTU_REDUCED_DEFERRED,
477	addr: &sk->sk_tsq_flags))
478	sock_hold(sk);
479	goto out;
480	}
481
482
483	/* Might be for an request_sock */
484	switch (sk->sk_state) {
485	case TCP_SYN_SENT:
486	case TCP_SYN_RECV:
487	/* Only in fast or simultaneous open. If a fast open socket is
488	* already accepted it is treated as a connected one below.
489	*/
490	if (fastopen && !fastopen->sk)
491	break;
492
493	ipv6_icmp_error(sk, skb, err, port: th->dest, ntohl(info), payload: (u8 *)th);
494
495	if (!sock_owned_by_user(sk))
496	tcp_done_with_error(sk, err);
497	else
498	WRITE_ONCE(sk->sk_err_soft, err);
499	goto out;
500	case TCP_LISTEN:
501	break;
502	default:
503	/* check if this ICMP message allows revert of backoff.
504	* (see RFC 6069)
505	*/
506	if (!fastopen && type == ICMPV6_DEST_UNREACH &&
507	code == ICMPV6_NOROUTE)
508	tcp_ld_RTO_revert(sk, seq);
509	}
510
511	if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
512	WRITE_ONCE(sk->sk_err, err);
513	sk_error_report(sk);
514	} else {
515	WRITE_ONCE(sk->sk_err_soft, err);
516	}
517	out:
518	bh_unlock_sock(sk);
519	sock_put(sk);
520	return 0;
521	}
522
523
524	static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
525	struct flowi *fl,
526	struct request_sock *req,
527	struct tcp_fastopen_cookie *foc,
528	enum tcp_synack_type synack_type,
529	struct sk_buff *syn_skb)
530	{
531	struct inet_request_sock *ireq = inet_rsk(sk: req);
532	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
533	struct ipv6_txoptions *opt;
534	struct flowi6 *fl6 = &fl->u.ip6;
535	struct sk_buff *skb;
536	int err = -ENOMEM;
537	u8 tclass;
538
539	/* First, grab a route. */
540	if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
541	IPPROTO_TCP)) == NULL)
542	goto done;
543
544	skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
545
546	if (skb) {
547	__tcp_v6_send_check(skb, saddr: &ireq->ir_v6_loc_addr,
548	daddr: &ireq->ir_v6_rmt_addr);
549
550	fl6->daddr = ireq->ir_v6_rmt_addr;
551	if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
552	fl6->flowlabel = ip6_flowlabel(hdr: ipv6_hdr(skb: ireq->pktopts));
553
554	tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
555	(tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
556	(np->tclass & INET_ECN_MASK) :
557	np->tclass;
558
559	if (!INET_ECN_is_capable(dsfield: tclass) &&
560	tcp_bpf_ca_needs_ecn(sk: (struct sock *)req))
561	tclass |= INET_ECN_ECT_0;
562
563	rcu_read_lock();
564	opt = ireq->ipv6_opt;
565	if (!opt)
566	opt = rcu_dereference(np->opt);
567	err = ip6_xmit(sk, skb, fl6, mark: skb->mark ? : READ_ONCE(sk->sk_mark),
568	opt, tclass, READ_ONCE(sk->sk_priority));
569	rcu_read_unlock();
570	err = net_xmit_eval(err);
571	}
572
573	done:
574	return err;
575	}
576
577
578	static void tcp_v6_reqsk_destructor(struct request_sock *req)
579	{
580	kfree(objp: inet_rsk(sk: req)->ipv6_opt);
581	consume_skb(skb: inet_rsk(sk: req)->pktopts);
582	}
583
584	#ifdef CONFIG_TCP_MD5SIG
585	static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
586	const struct in6_addr *addr,
587	int l3index)
588	{
589	return tcp_md5_do_lookup(sk, l3index,
590	addr: (union tcp_md5_addr *)addr, AF_INET6);
591	}
592
593	static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
594	const struct sock *addr_sk)
595	{
596	int l3index;
597
598	l3index = l3mdev_master_ifindex_by_index(net: sock_net(sk),
599	ifindex: addr_sk->sk_bound_dev_if);
600	return tcp_v6_md5_do_lookup(sk, addr: &addr_sk->sk_v6_daddr,
601	l3index);
602	}
603
604	static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
605	sockptr_t optval, int optlen)
606	{
607	struct tcp_md5sig cmd;
608	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
609	union tcp_ao_addr *addr;
610	int l3index = 0;
611	u8 prefixlen;
612	bool l3flag;
613	u8 flags;
614
615	if (optlen < sizeof(cmd))
616	return -EINVAL;
617
618	if (copy_from_sockptr(dst: &cmd, src: optval, size: sizeof(cmd)))
619	return -EFAULT;
620
621	if (sin6->sin6_family != AF_INET6)
622	return -EINVAL;
623
624	flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
625	l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
626
627	if (optname == TCP_MD5SIG_EXT &&
628	cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
629	prefixlen = cmd.tcpm_prefixlen;
630	if (prefixlen > 128 || (ipv6_addr_v4mapped(a: &sin6->sin6_addr) &&
631	prefixlen > 32))
632	return -EINVAL;
633	} else {
634	prefixlen = ipv6_addr_v4mapped(a: &sin6->sin6_addr) ? 32 : 128;
635	}
636
637	if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
638	cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
639	struct net_device *dev;
640
641	rcu_read_lock();
642	dev = dev_get_by_index_rcu(net: sock_net(sk), ifindex: cmd.tcpm_ifindex);
643	if (dev && netif_is_l3_master(dev))
644	l3index = dev->ifindex;
645	rcu_read_unlock();
646
647	/* ok to reference set/not set outside of rcu;
648	* right now device MUST be an L3 master
649	*/
650	if (!dev || !l3index)
651	return -EINVAL;
652	}
653
654	if (!cmd.tcpm_keylen) {
655	if (ipv6_addr_v4mapped(a: &sin6->sin6_addr))
656	return tcp_md5_do_del(sk, addr: (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
657	AF_INET, prefixlen,
658	l3index, flags);
659	return tcp_md5_do_del(sk, addr: (union tcp_md5_addr *)&sin6->sin6_addr,
660	AF_INET6, prefixlen, l3index, flags);
661	}
662
663	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
664	return -EINVAL;
665
666	if (ipv6_addr_v4mapped(a: &sin6->sin6_addr)) {
667	addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];
668
669	/* Don't allow keys for peers that have a matching TCP-AO key.
670	* See the comment in tcp_ao_add_cmd()
671	*/
672	if (tcp_ao_required(sk, saddr: addr, AF_INET,
673	l3index: l3flag ? l3index : -1, stat_inc: false))
674	return -EKEYREJECTED;
675	return tcp_md5_do_add(sk, addr,
676	AF_INET, prefixlen, l3index, flags,
677	newkey: cmd.tcpm_key, newkeylen: cmd.tcpm_keylen);
678	}
679
680	addr = (union tcp_md5_addr *)&sin6->sin6_addr;
681
682	/* Don't allow keys for peers that have a matching TCP-AO key.
683	* See the comment in tcp_ao_add_cmd()
684	*/
685	if (tcp_ao_required(sk, saddr: addr, AF_INET6, l3index: l3flag ? l3index : -1, stat_inc: false))
686	return -EKEYREJECTED;
687
688	return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
689	newkey: cmd.tcpm_key, newkeylen: cmd.tcpm_keylen);
690	}
691
692	static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp,
693	const struct in6_addr *daddr,
694	const struct in6_addr *saddr,
695	const struct tcphdr *th, int nbytes)
696	{
697	struct tcp6_pseudohdr *bp;
698	struct scatterlist sg;
699	struct tcphdr *_th;
700
701	bp = hp->scratch;
702	/* 1. TCP pseudo-header (RFC2460) */
703	bp->saddr = *saddr;
704	bp->daddr = *daddr;
705	bp->protocol = cpu_to_be32(IPPROTO_TCP);
706	bp->len = cpu_to_be32(nbytes);
707
708	_th = (struct tcphdr *)(bp + 1);
709	memcpy(_th, th, sizeof(*th));
710	_th->check = 0;
711
712	sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
713	ahash_request_set_crypt(req: hp->req, src: &sg, NULL,
714	nbytes: sizeof(*bp) + sizeof(*th));
715	return crypto_ahash_update(req: hp->req);
716	}
717
718	static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
719	const struct in6_addr *daddr, struct in6_addr *saddr,
720	const struct tcphdr *th)
721	{
722	struct tcp_sigpool hp;
723
724	if (tcp_sigpool_start(id: tcp_md5_sigpool_id, c: &hp))
725	goto clear_hash_nostart;
726
727	if (crypto_ahash_init(req: hp.req))
728	goto clear_hash;
729	if (tcp_v6_md5_hash_headers(hp: &hp, daddr, saddr, th, nbytes: th->doff << 2))
730	goto clear_hash;
731	if (tcp_md5_hash_key(hp: &hp, key))
732	goto clear_hash;
733	ahash_request_set_crypt(req: hp.req, NULL, result: md5_hash, nbytes: 0);
734	if (crypto_ahash_final(req: hp.req))
735	goto clear_hash;
736
737	tcp_sigpool_end(c: &hp);
738	return 0;
739
740	clear_hash:
741	tcp_sigpool_end(c: &hp);
742	clear_hash_nostart:
743	memset(md5_hash, 0, 16);
744	return 1;
745	}
746
747	static int tcp_v6_md5_hash_skb(char *md5_hash,
748	const struct tcp_md5sig_key *key,
749	const struct sock *sk,
750	const struct sk_buff *skb)
751	{
752	const struct tcphdr *th = tcp_hdr(skb);
753	const struct in6_addr *saddr, *daddr;
754	struct tcp_sigpool hp;
755
756	if (sk) { /* valid for establish/request sockets */
757	saddr = &sk->sk_v6_rcv_saddr;
758	daddr = &sk->sk_v6_daddr;
759	} else {
760	const struct ipv6hdr *ip6h = ipv6_hdr(skb);
761	saddr = &ip6h->saddr;
762	daddr = &ip6h->daddr;
763	}
764
765	if (tcp_sigpool_start(id: tcp_md5_sigpool_id, c: &hp))
766	goto clear_hash_nostart;
767
768	if (crypto_ahash_init(req: hp.req))
769	goto clear_hash;
770
771	if (tcp_v6_md5_hash_headers(hp: &hp, daddr, saddr, th, nbytes: skb->len))
772	goto clear_hash;
773	if (tcp_sigpool_hash_skb_data(hp: &hp, skb, header_len: th->doff << 2))
774	goto clear_hash;
775	if (tcp_md5_hash_key(hp: &hp, key))
776	goto clear_hash;
777	ahash_request_set_crypt(req: hp.req, NULL, result: md5_hash, nbytes: 0);
778	if (crypto_ahash_final(req: hp.req))
779	goto clear_hash;
780
781	tcp_sigpool_end(c: &hp);
782	return 0;
783
784	clear_hash:
785	tcp_sigpool_end(c: &hp);
786	clear_hash_nostart:
787	memset(md5_hash, 0, 16);
788	return 1;
789	}
790	#endif
791
792	static void tcp_v6_init_req(struct request_sock *req,
793	const struct sock *sk_listener,
794	struct sk_buff *skb,
795	u32 tw_isn)
796	{
797	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
798	struct inet_request_sock *ireq = inet_rsk(sk: req);
799	const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
800
801	ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
802	ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
803	ireq->ir_rmt_addr = LOOPBACK4_IPV6;
804	ireq->ir_loc_addr = LOOPBACK4_IPV6;
805
806	/* So that link locals have meaning */
807	if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
808	ipv6_addr_type(addr: &ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
809	ireq->ir_iif = tcp_v6_iif(skb);
810
811	if (!tw_isn &&
812	(ipv6_opt_accepted(sk: sk_listener, skb, opt: &TCP_SKB_CB(skb)->header.h6) ||
813	np->rxopt.bits.rxinfo ||
814	np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
815	np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
816	refcount_inc(r: &skb->users);
817	ireq->pktopts = skb;
818	}
819	}
820
821	static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
822	struct sk_buff *skb,
823	struct flowi *fl,
824	struct request_sock *req,
825	u32 tw_isn)
826	{
827	tcp_v6_init_req(req, sk_listener: sk, skb, tw_isn);
828
829	if (security_inet_conn_request(sk, skb, req))
830	return NULL;
831
832	return inet6_csk_route_req(sk, fl6: &fl->u.ip6, req, IPPROTO_TCP);
833	}
834
835	struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
836	.family = AF_INET6,
837	.obj_size = sizeof(struct tcp6_request_sock),
838	.rtx_syn_ack = tcp_rtx_synack,
839	.send_ack = tcp_v6_reqsk_send_ack,
840	.destructor = tcp_v6_reqsk_destructor,
841	.send_reset = tcp_v6_send_reset,
842	.syn_ack_timeout = tcp_syn_ack_timeout,
843	};
844
845	const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
846	.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) -
847	sizeof(struct ipv6hdr),
848	#ifdef CONFIG_TCP_MD5SIG
849	.req_md5_lookup = tcp_v6_md5_lookup,
850	.calc_md5_hash = tcp_v6_md5_hash_skb,
851	#endif
852	#ifdef CONFIG_TCP_AO
853	.ao_lookup = tcp_v6_ao_lookup_rsk,
854	.ao_calc_key = tcp_v6_ao_calc_key_rsk,
855	.ao_synack_hash = tcp_v6_ao_synack_hash,
856	#endif
857	#ifdef CONFIG_SYN_COOKIES
858	.cookie_init_seq = cookie_v6_init_sequence,
859	#endif
860	.route_req = tcp_v6_route_req,
861	.init_seq = tcp_v6_init_seq,
862	.init_ts_off = tcp_v6_init_ts_off,
863	.send_synack = tcp_v6_send_synack,
864	};
865
866	static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
867	u32 ack, u32 win, u32 tsval, u32 tsecr,
868	int oif, int rst, u8 tclass, __be32 label,
869	u32 priority, u32 txhash, struct tcp_key *key)
870	{
871	struct net *net = sk ? sock_net(sk) : dev_net_rcu(dev: skb_dst(skb)->dev);
872	unsigned int tot_len = sizeof(struct tcphdr);
873	struct sock *ctl_sk = net->ipv6.tcp_sk;
874	const struct tcphdr *th = tcp_hdr(skb);
875	__be32 mrst = 0, *topt;
876	struct dst_entry *dst;
877	struct sk_buff *buff;
878	struct tcphdr *t1;
879	struct flowi6 fl6;
880	u32 mark = 0;
881
882	if (tsecr)
883	tot_len += TCPOLEN_TSTAMP_ALIGNED;
884	if (tcp_key_is_md5(key))
885	tot_len += TCPOLEN_MD5SIG_ALIGNED;
886	if (tcp_key_is_ao(key))
887	tot_len += tcp_ao_len_aligned(key: key->ao_key);
888
889	#ifdef CONFIG_MPTCP
890	if (rst && !tcp_key_is_md5(key)) {
891	mrst = mptcp_reset_option(skb);
892
893	if (mrst)
894	tot_len += sizeof(__be32);
895	}
896	#endif
897
898	buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
899	if (!buff)
900	return;
901
902	skb_reserve(skb: buff, MAX_TCP_HEADER);
903
904	t1 = skb_push(skb: buff, len: tot_len);
905	skb_reset_transport_header(skb: buff);
906
907	/* Swap the send and the receive. */
908	memset(t1, 0, sizeof(*t1));
909	t1->dest = th->source;
910	t1->source = th->dest;
911	t1->doff = tot_len / 4;
912	t1->seq = htonl(seq);
913	t1->ack_seq = htonl(ack);
914	t1->ack = !rst || !th->ack;
915	t1->rst = rst;
916	t1->window = htons(win);
917
918	topt = (__be32 *)(t1 + 1);
919
920	if (tsecr) {
921	*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
922	(TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
923	*topt++ = htonl(tsval);
924	*topt++ = htonl(tsecr);
925	}
926
927	if (mrst)
928	*topt++ = mrst;
929
930	#ifdef CONFIG_TCP_MD5SIG
931	if (tcp_key_is_md5(key)) {
932	*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
933	(TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
934	tcp_v6_md5_hash_hdr(md5_hash: (__u8 *)topt, key: key->md5_key,
935	daddr: &ipv6_hdr(skb)->saddr,
936	saddr: &ipv6_hdr(skb)->daddr, th: t1);
937	}
938	#endif
939	#ifdef CONFIG_TCP_AO
940	if (tcp_key_is_ao(key)) {
941	*topt++ = htonl((TCPOPT_AO << 24) |
942	(tcp_ao_len(key->ao_key) << 16) |
943	(key->ao_key->sndid << 8) |
944	(key->rcv_next));
945
946	tcp_ao_hash_hdr(AF_INET6, ao_hash: (char *)topt, key: key->ao_key,
947	tkey: key->traffic_key,
948	daddr: (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
949	saddr: (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
950	th: t1, sne: key->sne);
951	}
952	#endif
953
954	memset(&fl6, 0, sizeof(fl6));
955	fl6.daddr = ipv6_hdr(skb)->saddr;
956	fl6.saddr = ipv6_hdr(skb)->daddr;
957	fl6.flowlabel = label;
958
959	buff->ip_summed = CHECKSUM_PARTIAL;
960
961	__tcp_v6_send_check(skb: buff, saddr: &fl6.saddr, daddr: &fl6.daddr);
962
963	fl6.flowi6_proto = IPPROTO_TCP;
964	if (rt6_need_strict(daddr: &fl6.daddr) && !oif)
965	fl6.flowi6_oif = tcp_v6_iif(skb);
966	else {
967	if (!oif && netif_index_is_l3_master(net, ifindex: skb->skb_iif))
968	oif = skb->skb_iif;
969
970	fl6.flowi6_oif = oif;
971	}
972
973	if (sk) {
974	/* unconstify the socket only to attach it to buff with care. */
975	skb_set_owner_edemux(skb: buff, sk: (struct sock *)sk);
976
977	if (sk->sk_state == TCP_TIME_WAIT)
978	mark = inet_twsk(sk)->tw_mark;
979	else
980	mark = READ_ONCE(sk->sk_mark);
981	skb_set_delivery_time(skb: buff, kt: tcp_transmit_time(sk), tstamp_type: SKB_CLOCK_MONOTONIC);
982	}
983	if (txhash) {
984	/* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
985	skb_set_hash(skb: buff, hash: txhash, type: PKT_HASH_TYPE_L4);
986	}
987	fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
988	fl6.fl6_dport = t1->dest;
989	fl6.fl6_sport = t1->source;
990	fl6.flowi6_uid = sock_net_uid(net, sk: sk && sk_fullsock(sk) ? sk : NULL);
991	security_skb_classify_flow(skb, flic: flowi6_to_flowi_common(fl6: &fl6));
992
993	/* Pass a socket to ip6_dst_lookup either it is for RST
994	* Underlying function will use this to retrieve the network
995	* namespace
996	*/
997	if (sk && sk->sk_state != TCP_TIME_WAIT)
998	dst = ip6_dst_lookup_flow(net, sk, fl6: &fl6, NULL); /*sk's xfrm_policy can be referred*/
999	else
1000	dst = ip6_dst_lookup_flow(net, sk: ctl_sk, fl6: &fl6, NULL);
1001	if (!IS_ERR(ptr: dst)) {
1002	skb_dst_set(skb: buff, dst);
1003	ip6_xmit(sk: ctl_sk, skb: buff, fl6: &fl6, mark: fl6.flowi6_mark, NULL,
1004	tclass, priority);
1005	TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
1006	if (rst)
1007	TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
1008	return;
1009	}
1010
1011	kfree_skb(skb: buff);
1012	}
1013
1014	static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
1015	enum sk_rst_reason reason)
1016	{
1017	const struct tcphdr *th = tcp_hdr(skb);
1018	struct ipv6hdr *ipv6h = ipv6_hdr(skb);
1019	const __u8 *md5_hash_location = NULL;
1020	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1021	bool allocated_traffic_key = false;
1022	#endif
1023	const struct tcp_ao_hdr *aoh;
1024	struct tcp_key key = {};
1025	u32 seq = 0, ack_seq = 0;
1026	__be32 label = 0;
1027	u32 priority = 0;
1028	struct net *net;
1029	u32 txhash = 0;
1030	int oif = 0;
1031	#ifdef CONFIG_TCP_MD5SIG
1032	unsigned char newhash[16];
1033	int genhash;
1034	struct sock *sk1 = NULL;
1035	#endif
1036
1037	if (th->rst)
1038	return;
1039
1040	/* If sk not NULL, it means we did a successful lookup and incoming
1041	* route had to be correct. prequeue might have dropped our dst.
1042	*/
1043	if (!sk && !ipv6_unicast_destination(skb))
1044	return;
1045
1046	net = sk ? sock_net(sk) : dev_net_rcu(dev: skb_dst(skb)->dev);
1047	/* Invalid TCP option size or twice included auth */
1048	if (tcp_parse_auth_options(th, md5_hash: &md5_hash_location, aoh: &aoh))
1049	return;
1050	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1051	rcu_read_lock();
1052	#endif
1053	#ifdef CONFIG_TCP_MD5SIG
1054	if (sk && sk_fullsock(sk)) {
1055	int l3index;
1056
1057	/* sdif set, means packet ingressed via a device
1058	* in an L3 domain and inet_iif is set to it.
1059	*/
1060	l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1061	key.md5_key = tcp_v6_md5_do_lookup(sk, addr: &ipv6h->saddr, l3index);
1062	if (key.md5_key)
1063	key.type = TCP_KEY_MD5;
1064	} else if (md5_hash_location) {
1065	int dif = tcp_v6_iif_l3_slave(skb);
1066	int sdif = tcp_v6_sdif(skb);
1067	int l3index;
1068
1069	/*
1070	* active side is lost. Try to find listening socket through
1071	* source port, and then find md5 key through listening socket.
1072	* we are not loose security here:
1073	* Incoming packet is checked with md5 hash with finding key,
1074	* no RST generated if md5 hash doesn't match.
1075	*/
1076	sk1 = inet6_lookup_listener(net, hashinfo: net->ipv4.tcp_death_row.hashinfo,
1077	NULL, doff: 0, saddr: &ipv6h->saddr, sport: th->source,
1078	daddr: &ipv6h->daddr, ntohs(th->source),
1079	dif, sdif);
1080	if (!sk1)
1081	goto out;
1082
1083	/* sdif set, means packet ingressed via a device
1084	* in an L3 domain and dif is set to it.
1085	*/
1086	l3index = tcp_v6_sdif(skb) ? dif : 0;
1087
1088	key.md5_key = tcp_v6_md5_do_lookup(sk: sk1, addr: &ipv6h->saddr, l3index);
1089	if (!key.md5_key)
1090	goto out;
1091	key.type = TCP_KEY_MD5;
1092
1093	genhash = tcp_v6_md5_hash_skb(md5_hash: newhash, key: key.md5_key, NULL, skb);
1094	if (genhash || memcmp(p: md5_hash_location, q: newhash, size: 16) != 0)
1095	goto out;
1096	}
1097	#endif
1098
1099	if (th->ack)
1100	seq = ntohl(th->ack_seq);
1101	else
1102	ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1103	(th->doff << 2);
1104
1105	#ifdef CONFIG_TCP_AO
1106	if (aoh) {
1107	int l3index;
1108
1109	l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1110	if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
1111	key: &key.ao_key, traffic_key: &key.traffic_key,
1112	allocated_traffic_key: &allocated_traffic_key,
1113	keyid: &key.rcv_next, sne: &key.sne))
1114	goto out;
1115	key.type = TCP_KEY_AO;
1116	}
1117	#endif
1118
1119	if (sk) {
1120	oif = sk->sk_bound_dev_if;
1121	if (sk_fullsock(sk)) {
1122	if (inet6_test_bit(REPFLOW, sk))
1123	label = ip6_flowlabel(hdr: ipv6h);
1124	priority = READ_ONCE(sk->sk_priority);
1125	txhash = sk->sk_txhash;
1126	}
1127	if (sk->sk_state == TCP_TIME_WAIT) {
1128	label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1129	priority = inet_twsk(sk)->tw_priority;
1130	txhash = inet_twsk(sk)->tw_txhash;
1131	}
1132	} else {
1133	if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
1134	label = ip6_flowlabel(hdr: ipv6h);
1135	}
1136
1137	trace_tcp_send_reset(sk, skb__nullable: skb, reason);
1138
1139	tcp_v6_send_response(sk, skb, seq, ack: ack_seq, win: 0, tsval: 0, tsecr: 0, oif, rst: 1,
1140	tclass: ipv6_get_dsfield(ipv6h) & ~INET_ECN_MASK,
1141	label, priority, txhash,
1142	key: &key);
1143
1144	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1145	out:
1146	if (allocated_traffic_key)
1147	kfree(objp: key.traffic_key);
1148	rcu_read_unlock();
1149	#endif
1150	}
1151
1152	static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1153	u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1154	struct tcp_key *key, u8 tclass,
1155	__be32 label, u32 priority, u32 txhash)
1156	{
1157	tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, rst: 0,
1158	tclass, label, priority, txhash, key);
1159	}
1160
1161	static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb,
1162	enum tcp_tw_status tw_status)
1163	{
1164	struct inet_timewait_sock *tw = inet_twsk(sk);
1165	struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1166	u8 tclass = tw->tw_tclass;
1167	struct tcp_key key = {};
1168
1169	if (tw_status == TCP_TW_ACK_OOW)
1170	tclass &= ~INET_ECN_MASK;
1171	#ifdef CONFIG_TCP_AO
1172	struct tcp_ao_info *ao_info;
1173
1174	if (static_branch_unlikely(&tcp_ao_needed.key)) {
1175
1176	/* FIXME: the segment to-be-acked is not verified yet */
1177	ao_info = rcu_dereference(tcptw->ao_info);
1178	if (ao_info) {
1179	const struct tcp_ao_hdr *aoh;
1180
1181	/* Invalid TCP option size or twice included auth */
1182	if (tcp_parse_auth_options(th: tcp_hdr(skb), NULL, aoh: &aoh))
1183	goto out;
1184	if (aoh)
1185	key.ao_key = tcp_ao_established_key(sk, ao: ao_info,
1186	sndid: aoh->rnext_keyid, rcvid: -1);
1187	}
1188	}
1189	if (key.ao_key) {
1190	struct tcp_ao_key *rnext_key;
1191
1192	key.traffic_key = snd_other_key(key: key.ao_key);
1193	/* rcv_next switches to our rcv_next */
1194	rnext_key = READ_ONCE(ao_info->rnext_key);
1195	key.rcv_next = rnext_key->rcvid;
1196	key.sne = READ_ONCE(ao_info->snd_sne);
1197	key.type = TCP_KEY_AO;
1198	#else
1199	if (0) {
1200	#endif
1201	#ifdef CONFIG_TCP_MD5SIG
1202	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1203	key.md5_key = tcp_twsk_md5_key(tcptw);
1204	if (key.md5_key)
1205	key.type = TCP_KEY_MD5;
1206	#endif
1207	}
1208
1209	tcp_v6_send_ack(sk, skb, seq: tcptw->tw_snd_nxt,
1210	READ_ONCE(tcptw->tw_rcv_nxt),
1211	win: tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1212	tsval: tcp_tw_tsval(tcptw),
1213	READ_ONCE(tcptw->tw_ts_recent), oif: tw->tw_bound_dev_if,
1214	key: &key, tclass, cpu_to_be32(tw->tw_flowlabel),
1215	priority: tw->tw_priority, txhash: tw->tw_txhash);
1216
1217	#ifdef CONFIG_TCP_AO
1218	out:
1219	#endif
1220	inet_twsk_put(tw);
1221	}
1222
1223	static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1224	struct request_sock *req)
1225	{
1226	struct tcp_key key = {};
1227
1228	#ifdef CONFIG_TCP_AO
1229	if (static_branch_unlikely(&tcp_ao_needed.key) &&
1230	tcp_rsk_used_ao(req)) {
1231	const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
1232	const struct tcp_ao_hdr *aoh;
1233	int l3index;
1234
1235	l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1236	/* Invalid TCP option size or twice included auth */
1237	if (tcp_parse_auth_options(th: tcp_hdr(skb), NULL, aoh: &aoh))
1238	return;
1239	if (!aoh)
1240	return;
1241	key.ao_key = tcp_ao_do_lookup(sk, l3index,
1242	addr: (union tcp_ao_addr *)addr,
1243	AF_INET6, sndid: aoh->rnext_keyid, rcvid: -1);
1244	if (unlikely(!key.ao_key)) {
1245	/* Send ACK with any matching MKT for the peer */
1246	key.ao_key = tcp_ao_do_lookup(sk, l3index,
1247	addr: (union tcp_ao_addr *)addr,
1248	AF_INET6, sndid: -1, rcvid: -1);
1249	/* Matching key disappeared (user removed the key?)
1250	* let the handshake timeout.
1251	*/
1252	if (!key.ao_key) {
1253	net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
1254	addr,
1255	ntohs(tcp_hdr(skb)->source),
1256	&ipv6_hdr(skb)->daddr,
1257	ntohs(tcp_hdr(skb)->dest));
1258	return;
1259	}
1260	}
1261	key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1262	if (!key.traffic_key)
1263	return;
1264
1265	key.type = TCP_KEY_AO;
1266	key.rcv_next = aoh->keyid;
1267	tcp_v6_ao_calc_key_rsk(mkt: key.ao_key, key: key.traffic_key, req);
1268	#else
1269	if (0) {
1270	#endif
1271	#ifdef CONFIG_TCP_MD5SIG
1272	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1273	int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1274
1275	key.md5_key = tcp_v6_md5_do_lookup(sk, addr: &ipv6_hdr(skb)->saddr,
1276	l3index);
1277	if (key.md5_key)
1278	key.type = TCP_KEY_MD5;
1279	#endif
1280	}
1281
1282	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1283	* sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1284	*/
1285	tcp_v6_send_ack(sk, skb, seq: (sk->sk_state == TCP_LISTEN) ?
1286	tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1287	ack: tcp_rsk(req)->rcv_nxt,
1288	win: tcp_synack_window(req) >> inet_rsk(sk: req)->rcv_wscale,
1289	tsval: tcp_rsk_tsval(treq: tcp_rsk(req)),
1290	tsecr: req->ts_recent, oif: sk->sk_bound_dev_if,
1291	key: &key, tclass: ipv6_get_dsfield(ipv6h: ipv6_hdr(skb)) & ~INET_ECN_MASK,
1292	label: 0,
1293	READ_ONCE(sk->sk_priority),
1294	READ_ONCE(tcp_rsk(req)->txhash));
1295	if (tcp_key_is_ao(key: &key))
1296	kfree(objp: key.traffic_key);
1297	}
1298
1299
1300	static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1301	{
1302	#ifdef CONFIG_SYN_COOKIES
1303	const struct tcphdr *th = tcp_hdr(skb);
1304
1305	if (!th->syn)
1306	sk = cookie_v6_check(sk, skb);
1307	#endif
1308	return sk;
1309	}
1310
1311	u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1312	struct tcphdr *th, u32 *cookie)
1313	{
1314	u16 mss = 0;
1315	#ifdef CONFIG_SYN_COOKIES
1316	mss = tcp_get_syncookie_mss(rsk_ops: &tcp6_request_sock_ops,
1317	af_ops: &tcp_request_sock_ipv6_ops, sk, th);
1318	if (mss) {
1319	*cookie = __cookie_v6_init_sequence(iph, th, mssp: &mss);
1320	tcp_synq_overflow(sk);
1321	}
1322	#endif
1323	return mss;
1324	}
1325
1326	static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1327	{
1328	if (skb->protocol == htons(ETH_P_IP))
1329	return tcp_v4_conn_request(sk, skb);
1330
1331	if (!ipv6_unicast_destination(skb))
1332	goto drop;
1333
1334	if (ipv6_addr_v4mapped(a: &ipv6_hdr(skb)->saddr)) {
1335	__IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1336	return 0;
1337	}
1338
1339	return tcp_conn_request(rsk_ops: &tcp6_request_sock_ops,
1340	af_ops: &tcp_request_sock_ipv6_ops, sk, skb);
1341
1342	drop:
1343	tcp_listendrop(sk);
1344	return 0; /* don't send reset */
1345	}
1346
1347	static void tcp_v6_restore_cb(struct sk_buff *skb)
1348	{
1349	/* We need to move header back to the beginning if xfrm6_policy_check()
1350	* and tcp_v6_fill_cb() are going to be called again.
1351	* ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1352	*/
1353	memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1354	sizeof(struct inet6_skb_parm));
1355	}
1356
1357	static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1358	struct request_sock *req,
1359	struct dst_entry *dst,
1360	struct request_sock *req_unhash,
1361	bool *own_req)
1362	{
1363	struct inet_request_sock *ireq;
1364	struct ipv6_pinfo *newnp;
1365	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1366	struct ipv6_txoptions *opt;
1367	struct inet_sock *newinet;
1368	bool found_dup_sk = false;
1369	struct tcp_sock *newtp;
1370	struct sock *newsk;
1371	#ifdef CONFIG_TCP_MD5SIG
1372	struct tcp_md5sig_key *key;
1373	int l3index;
1374	#endif
1375	struct flowi6 fl6;
1376
1377	if (skb->protocol == htons(ETH_P_IP)) {
1378	/*
1379	* v6 mapped
1380	*/
1381
1382	newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst,
1383	req_unhash, own_req);
1384
1385	if (!newsk)
1386	return NULL;
1387
1388	inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1389
1390	newnp = tcp_inet6_sk(newsk);
1391	newtp = tcp_sk(newsk);
1392
1393	memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1394
1395	newnp->saddr = newsk->sk_v6_rcv_saddr;
1396
1397	inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1398	if (sk_is_mptcp(sk: newsk))
1399	mptcpv6_handle_mapped(sk: newsk, mapped: true);
1400	newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1401	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1402	newtp->af_specific = &tcp_sock_ipv6_mapped_specific;
1403	#endif
1404
1405	newnp->ipv6_mc_list = NULL;
1406	newnp->ipv6_ac_list = NULL;
1407	newnp->ipv6_fl_list = NULL;
1408	newnp->pktoptions = NULL;
1409	newnp->opt = NULL;
1410	newnp->mcast_oif = inet_iif(skb);
1411	newnp->mcast_hops = ip_hdr(skb)->ttl;
1412	newnp->rcv_flowinfo = 0;
1413	if (inet6_test_bit(REPFLOW, sk))
1414	newnp->flow_label = 0;
1415
1416	/*
1417	* No need to charge this sock to the relevant IPv6 refcnt debug socks count
1418	* here, tcp_create_openreq_child now does this for us, see the comment in
1419	* that function for the gory details. -acme
1420	*/
1421
1422	/* It is tricky place. Until this moment IPv4 tcp
1423	worked with IPv6 icsk.icsk_af_ops.
1424	Sync it now.
1425	*/
1426	tcp_sync_mss(sk: newsk, inet_csk(newsk)->icsk_pmtu_cookie);
1427
1428	return newsk;
1429	}
1430
1431	ireq = inet_rsk(sk: req);
1432
1433	if (sk_acceptq_is_full(sk))
1434	goto out_overflow;
1435
1436	if (!dst) {
1437	dst = inet6_csk_route_req(sk, fl6: &fl6, req, IPPROTO_TCP);
1438	if (!dst)
1439	goto out;
1440	}
1441
1442	newsk = tcp_create_openreq_child(sk, req, skb);
1443	if (!newsk)
1444	goto out_nonewsk;
1445
1446	/*
1447	* No need to charge this sock to the relevant IPv6 refcnt debug socks
1448	* count here, tcp_create_openreq_child now does this for us, see the
1449	* comment in that function for the gory details. -acme
1450	*/
1451
1452	newsk->sk_gso_type = SKB_GSO_TCPV6;
1453	inet6_sk_rx_dst_set(sk: newsk, skb);
1454
1455	inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1456
1457	newtp = tcp_sk(newsk);
1458	newinet = inet_sk(newsk);
1459	newnp = tcp_inet6_sk(newsk);
1460
1461	memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1462
1463	ip6_dst_store(sk: newsk, dst, NULL, NULL);
1464
1465	newnp->saddr = ireq->ir_v6_loc_addr;
1466
1467	/* Now IPv6 options...
1468
1469	First: no IPv4 options.
1470	*/
1471	newinet->inet_opt = NULL;
1472	newnp->ipv6_mc_list = NULL;
1473	newnp->ipv6_ac_list = NULL;
1474	newnp->ipv6_fl_list = NULL;
1475
1476	/* Clone RX bits */
1477	newnp->rxopt.all = np->rxopt.all;
1478
1479	newnp->pktoptions = NULL;
1480	newnp->opt = NULL;
1481	newnp->mcast_oif = tcp_v6_iif(skb);
1482	newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1483	newnp->rcv_flowinfo = ip6_flowinfo(hdr: ipv6_hdr(skb));
1484	if (inet6_test_bit(REPFLOW, sk))
1485	newnp->flow_label = ip6_flowlabel(hdr: ipv6_hdr(skb));
1486
1487	/* Set ToS of the new socket based upon the value of incoming SYN.
1488	* ECT bits are set later in tcp_init_transfer().
1489	*/
1490	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1491	newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1492
1493	/* Clone native IPv6 options from listening socket (if any)
1494
1495	Yes, keeping reference count would be much more clever,
1496	but we make one more one thing there: reattach optmem
1497	to newsk.
1498	*/
1499	opt = ireq->ipv6_opt;
1500	if (!opt)
1501	opt = rcu_dereference(np->opt);
1502	if (opt) {
1503	opt = ipv6_dup_options(sk: newsk, opt);
1504	RCU_INIT_POINTER(newnp->opt, opt);
1505	}
1506	inet_csk(newsk)->icsk_ext_hdr_len = 0;
1507	if (opt)
1508	inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1509	opt->opt_flen;
1510
1511	tcp_ca_openreq_child(sk: newsk, dst);
1512
1513	tcp_sync_mss(sk: newsk, pmtu: dst_mtu(dst));
1514	newtp->advmss = tcp_mss_clamp(tcp_sk(sk), mss: dst_metric_advmss(dst));
1515
1516	tcp_initialize_rcv_mss(sk: newsk);
1517
1518	#ifdef CONFIG_TCP_MD5SIG
1519	l3index = l3mdev_master_ifindex_by_index(net: sock_net(sk), ifindex: ireq->ir_iif);
1520
1521	if (!tcp_rsk_used_ao(req)) {
1522	/* Copy over the MD5 key from the original socket */
1523	key = tcp_v6_md5_do_lookup(sk, addr: &newsk->sk_v6_daddr, l3index);
1524	if (key) {
1525	const union tcp_md5_addr *addr;
1526
1527	addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
1528	if (tcp_md5_key_copy(sk: newsk, addr, AF_INET6, prefixlen: 128, l3index, key)) {
1529	inet_csk_prepare_forced_close(sk: newsk);
1530	tcp_done(sk: newsk);
1531	goto out;
1532	}
1533	}
1534	}
1535	#endif
1536	#ifdef CONFIG_TCP_AO
1537	/* Copy over tcp_ao_info if any */
1538	if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
1539	goto out; /* OOM */
1540	#endif
1541
1542	if (__inet_inherit_port(sk, child: newsk) < 0) {
1543	inet_csk_prepare_forced_close(sk: newsk);
1544	tcp_done(sk: newsk);
1545	goto out;
1546	}
1547	*own_req = inet_ehash_nolisten(sk: newsk, osk: req_to_sk(req: req_unhash),
1548	found_dup_sk: &found_dup_sk);
1549	if (*own_req) {
1550	tcp_move_syn(tp: newtp, req);
1551
1552	/* Clone pktoptions received with SYN, if we own the req */
1553	if (ireq->pktopts) {
1554	newnp->pktoptions = skb_clone_and_charge_r(skb: ireq->pktopts, sk: newsk);
1555	consume_skb(skb: ireq->pktopts);
1556	ireq->pktopts = NULL;
1557	if (newnp->pktoptions)
1558	tcp_v6_restore_cb(skb: newnp->pktoptions);
1559	}
1560	} else {
1561	if (!req_unhash && found_dup_sk) {
1562	/* This code path should only be executed in the
1563	* syncookie case only
1564	*/
1565	bh_unlock_sock(newsk);
1566	sock_put(sk: newsk);
1567	newsk = NULL;
1568	}
1569	}
1570
1571	return newsk;
1572
1573	out_overflow:
1574	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1575	out_nonewsk:
1576	dst_release(dst);
1577	out:
1578	tcp_listendrop(sk);
1579	return NULL;
1580	}
1581
1582	INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1583	u32));
1584	/* The socket must have it's spinlock held when we get
1585	* here, unless it is a TCP_LISTEN socket.
1586	*
1587	* We have a potential double-lock case here, so even when
1588	* doing backlog processing we use the BH locking scheme.
1589	* This is because we cannot sleep with the original spinlock
1590	* held.
1591	*/
1592	INDIRECT_CALLABLE_SCOPE
1593	int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1594	{
1595	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1596	struct sk_buff *opt_skb = NULL;
1597	enum skb_drop_reason reason;
1598	struct tcp_sock *tp;
1599
1600	/* Imagine: socket is IPv6. IPv4 packet arrives,
1601	goes to IPv4 receive handler and backlogged.
1602	From backlog it always goes here. Kerboom...
1603	Fortunately, tcp_rcv_established and rcv_established
1604	handle them correctly, but it is not case with
1605	tcp_v6_hnd_req and tcp_v6_send_reset(). --ANK
1606	*/
1607
1608	if (skb->protocol == htons(ETH_P_IP))
1609	return tcp_v4_do_rcv(sk, skb);
1610
1611	/*
1612	* socket locking is here for SMP purposes as backlog rcv
1613	* is currently called with bh processing disabled.
1614	*/
1615
1616	/* Do Stevens' IPV6_PKTOPTIONS.
1617
1618	Yes, guys, it is the only place in our code, where we
1619	may make it not affecting IPv4.
1620	The rest of code is protocol independent,
1621	and I do not like idea to uglify IPv4.
1622
1623	Actually, all the idea behind IPV6_PKTOPTIONS
1624	looks not very well thought. For now we latch
1625	options, received in the last packet, enqueued
1626	by tcp. Feel free to propose better solution.
1627	--ANK (980728)
1628	*/
1629	if (np->rxopt.all && sk->sk_state != TCP_LISTEN)
1630	opt_skb = skb_clone_and_charge_r(skb, sk);
1631
1632	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1633	struct dst_entry *dst;
1634
1635	dst = rcu_dereference_protected(sk->sk_rx_dst,
1636	lockdep_sock_is_held(sk));
1637
1638	sock_rps_save_rxhash(sk, skb);
1639	sk_mark_napi_id(sk, skb);
1640	if (dst) {
1641	if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1642	INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1643	dst, sk->sk_rx_dst_cookie) == NULL) {
1644	RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1645	dst_release(dst);
1646	}
1647	}
1648
1649	tcp_rcv_established(sk, skb);
1650	if (opt_skb)
1651	goto ipv6_pktoptions;
1652	return 0;
1653	}
1654
1655	if (tcp_checksum_complete(skb))
1656	goto csum_err;
1657
1658	if (sk->sk_state == TCP_LISTEN) {
1659	struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1660
1661	if (nsk != sk) {
1662	if (nsk) {
1663	reason = tcp_child_process(parent: sk, child: nsk, skb);
1664	if (reason)
1665	goto reset;
1666	}
1667	return 0;
1668	}
1669	} else
1670	sock_rps_save_rxhash(sk, skb);
1671
1672	reason = tcp_rcv_state_process(sk, skb);
1673	if (reason)
1674	goto reset;
1675	if (opt_skb)
1676	goto ipv6_pktoptions;
1677	return 0;
1678
1679	reset:
1680	tcp_v6_send_reset(sk, skb, reason: sk_rst_convert_drop_reason(reason));
1681	discard:
1682	if (opt_skb)
1683	__kfree_skb(skb: opt_skb);
1684	sk_skb_reason_drop(sk, skb, reason);
1685	return 0;
1686	csum_err:
1687	reason = SKB_DROP_REASON_TCP_CSUM;
1688	trace_tcp_bad_csum(skb);
1689	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1690	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1691	goto discard;
1692
1693
1694	ipv6_pktoptions:
1695	/* Do you ask, what is it?
1696
1697	1. skb was enqueued by tcp.
1698	2. skb is added to tail of read queue, rather than out of order.
1699	3. socket is not in passive state.
1700	4. Finally, it really contains options, which user wants to receive.
1701	*/
1702	tp = tcp_sk(sk);
1703	if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1704	!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1705	if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1706	WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb));
1707	if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1708	WRITE_ONCE(np->mcast_hops,
1709	ipv6_hdr(opt_skb)->hop_limit);
1710	if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1711	np->rcv_flowinfo = ip6_flowinfo(hdr: ipv6_hdr(skb: opt_skb));
1712	if (inet6_test_bit(REPFLOW, sk))
1713	np->flow_label = ip6_flowlabel(hdr: ipv6_hdr(skb: opt_skb));
1714	if (ipv6_opt_accepted(sk, skb: opt_skb, opt: &TCP_SKB_CB(opt_skb)->header.h6)) {
1715	tcp_v6_restore_cb(skb: opt_skb);
1716	opt_skb = xchg(&np->pktoptions, opt_skb);
1717	} else {
1718	__kfree_skb(skb: opt_skb);
1719	opt_skb = xchg(&np->pktoptions, NULL);
1720	}
1721	}
1722
1723	consume_skb(skb: opt_skb);
1724	return 0;
1725	}
1726
1727	static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1728	const struct tcphdr *th)
1729	{
1730	/* This is tricky: we move IP6CB at its correct location into
1731	* TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1732	* _decode_session6() uses IP6CB().
1733	* barrier() makes sure compiler won't play aliasing games.
1734	*/
1735	memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1736	sizeof(struct inet6_skb_parm));
1737	barrier();
1738
1739	TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1740	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1741	skb->len - th->doff*4);
1742	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1743	TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th);
1744	TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(ipv6h: hdr);
1745	TCP_SKB_CB(skb)->sacked = 0;
1746	TCP_SKB_CB(skb)->has_rxtstamp =
1747	skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1748	}
1749
1750	INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1751	{
1752	struct net *net = dev_net_rcu(dev: skb->dev);
1753	enum skb_drop_reason drop_reason;
1754	enum tcp_tw_status tw_status;
1755	int sdif = inet6_sdif(skb);
1756	int dif = inet6_iif(skb);
1757	const struct tcphdr *th;
1758	const struct ipv6hdr *hdr;
1759	struct sock *sk = NULL;
1760	bool refcounted;
1761	int ret;
1762	u32 isn;
1763
1764	drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1765	if (skb->pkt_type != PACKET_HOST)
1766	goto discard_it;
1767
1768	/*
1769	* Count it even if it's bad.
1770	*/
1771	__TCP_INC_STATS(net, TCP_MIB_INSEGS);
1772
1773	if (!pskb_may_pull(skb, len: sizeof(struct tcphdr)))
1774	goto discard_it;
1775
1776	th = (const struct tcphdr *)skb->data;
1777
1778	if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1779	drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1780	goto bad_packet;
1781	}
1782	if (!pskb_may_pull(skb, len: th->doff*4))
1783	goto discard_it;
1784
1785	if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1786	goto csum_error;
1787
1788	th = (const struct tcphdr *)skb->data;
1789	hdr = ipv6_hdr(skb);
1790
1791	lookup:
1792	sk = __inet6_lookup_skb(hashinfo: net->ipv4.tcp_death_row.hashinfo, skb, doff: __tcp_hdrlen(th),
1793	sport: th->source, dport: th->dest, iif: inet6_iif(skb), sdif,
1794	refcounted: &refcounted);
1795	if (!sk)
1796	goto no_tcp_socket;
1797
1798	if (sk->sk_state == TCP_TIME_WAIT)
1799	goto do_time_wait;
1800
1801	if (sk->sk_state == TCP_NEW_SYN_RECV) {
1802	struct request_sock *req = inet_reqsk(sk);
1803	bool req_stolen = false;
1804	struct sock *nsk;
1805
1806	sk = req->rsk_listener;
1807	if (!xfrm6_policy_check(sk, dir: XFRM_POLICY_IN, skb))
1808	drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1809	else
1810	drop_reason = tcp_inbound_hash(sk, req, skb,
1811	saddr: &hdr->saddr, daddr: &hdr->daddr,
1812	AF_INET6, dif, sdif);
1813	if (drop_reason) {
1814	sk_drops_add(sk, skb);
1815	reqsk_put(req);
1816	goto discard_it;
1817	}
1818	if (tcp_checksum_complete(skb)) {
1819	reqsk_put(req);
1820	goto csum_error;
1821	}
1822	if (unlikely(sk->sk_state != TCP_LISTEN)) {
1823	nsk = reuseport_migrate_sock(sk, migrating_sk: req_to_sk(req), skb);
1824	if (!nsk) {
1825	inet_csk_reqsk_queue_drop_and_put(sk, req);
1826	goto lookup;
1827	}
1828	sk = nsk;
1829	/* reuseport_migrate_sock() has already held one sk_refcnt
1830	* before returning.
1831	*/
1832	} else {
1833	sock_hold(sk);
1834	}
1835	refcounted = true;
1836	nsk = NULL;
1837	if (!tcp_filter(sk, skb)) {
1838	th = (const struct tcphdr *)skb->data;
1839	hdr = ipv6_hdr(skb);
1840	tcp_v6_fill_cb(skb, hdr, th);
1841	nsk = tcp_check_req(sk, skb, req, fastopen: false, lost_race: &req_stolen,
1842	drop_reason: &drop_reason);
1843	} else {
1844	drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1845	}
1846	if (!nsk) {
1847	reqsk_put(req);
1848	if (req_stolen) {
1849	/* Another cpu got exclusive access to req
1850	* and created a full blown socket.
1851	* Try to feed this packet to this socket
1852	* instead of discarding it.
1853	*/
1854	tcp_v6_restore_cb(skb);
1855	sock_put(sk);
1856	goto lookup;
1857	}
1858	goto discard_and_relse;
1859	}
1860	nf_reset_ct(skb);
1861	if (nsk == sk) {
1862	reqsk_put(req);
1863	tcp_v6_restore_cb(skb);
1864	} else {
1865	drop_reason = tcp_child_process(parent: sk, child: nsk, skb);
1866	if (drop_reason) {
1867	enum sk_rst_reason rst_reason;
1868
1869	rst_reason = sk_rst_convert_drop_reason(reason: drop_reason);
1870	tcp_v6_send_reset(sk: nsk, skb, reason: rst_reason);
1871	goto discard_and_relse;
1872	}
1873	sock_put(sk);
1874	return 0;
1875	}
1876	}
1877
1878	process:
1879	if (static_branch_unlikely(&ip6_min_hopcount)) {
1880	/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
1881	if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
1882	__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1883	drop_reason = SKB_DROP_REASON_TCP_MINTTL;
1884	goto discard_and_relse;
1885	}
1886	}
1887
1888	if (!xfrm6_policy_check(sk, dir: XFRM_POLICY_IN, skb)) {
1889	drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1890	goto discard_and_relse;
1891	}
1892
1893	drop_reason = tcp_inbound_hash(sk, NULL, skb, saddr: &hdr->saddr, daddr: &hdr->daddr,
1894	AF_INET6, dif, sdif);
1895	if (drop_reason)
1896	goto discard_and_relse;
1897
1898	nf_reset_ct(skb);
1899
1900	if (tcp_filter(sk, skb)) {
1901	drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1902	goto discard_and_relse;
1903	}
1904	th = (const struct tcphdr *)skb->data;
1905	hdr = ipv6_hdr(skb);
1906	tcp_v6_fill_cb(skb, hdr, th);
1907
1908	skb->dev = NULL;
1909
1910	if (sk->sk_state == TCP_LISTEN) {
1911	ret = tcp_v6_do_rcv(sk, skb);
1912	goto put_and_return;
1913	}
1914
1915	sk_incoming_cpu_update(sk);
1916
1917	bh_lock_sock_nested(sk);
1918	tcp_segs_in(tcp_sk(sk), skb);
1919	ret = 0;
1920	if (!sock_owned_by_user(sk)) {
1921	ret = tcp_v6_do_rcv(sk, skb);
1922	} else {
1923	if (tcp_add_backlog(sk, skb, reason: &drop_reason))
1924	goto discard_and_relse;
1925	}
1926	bh_unlock_sock(sk);
1927	put_and_return:
1928	if (refcounted)
1929	sock_put(sk);
1930	return ret ? -1 : 0;
1931
1932	no_tcp_socket:
1933	drop_reason = SKB_DROP_REASON_NO_SOCKET;
1934	if (!xfrm6_policy_check(NULL, dir: XFRM_POLICY_IN, skb))
1935	goto discard_it;
1936
1937	tcp_v6_fill_cb(skb, hdr, th);
1938
1939	if (tcp_checksum_complete(skb)) {
1940	csum_error:
1941	drop_reason = SKB_DROP_REASON_TCP_CSUM;
1942	trace_tcp_bad_csum(skb);
1943	__TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1944	bad_packet:
1945	__TCP_INC_STATS(net, TCP_MIB_INERRS);
1946	} else {
1947	tcp_v6_send_reset(NULL, skb, reason: sk_rst_convert_drop_reason(reason: drop_reason));
1948	}
1949
1950	discard_it:
1951	SKB_DR_OR(drop_reason, NOT_SPECIFIED);
1952	sk_skb_reason_drop(sk, skb, reason: drop_reason);
1953	return 0;
1954
1955	discard_and_relse:
1956	sk_drops_add(sk, skb);
1957	if (refcounted)
1958	sock_put(sk);
1959	goto discard_it;
1960
1961	do_time_wait:
1962	if (!xfrm6_policy_check(NULL, dir: XFRM_POLICY_IN, skb)) {
1963	drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1964	inet_twsk_put(tw: inet_twsk(sk));
1965	goto discard_it;
1966	}
1967
1968	tcp_v6_fill_cb(skb, hdr, th);
1969
1970	if (tcp_checksum_complete(skb)) {
1971	inet_twsk_put(tw: inet_twsk(sk));
1972	goto csum_error;
1973	}
1974
1975	tw_status = tcp_timewait_state_process(tw: inet_twsk(sk), skb, th, tw_isn: &isn,
1976	drop_reason: &drop_reason);
1977	switch (tw_status) {
1978	case TCP_TW_SYN:
1979	{
1980	struct sock *sk2;
1981
1982	sk2 = inet6_lookup_listener(net, hashinfo: net->ipv4.tcp_death_row.hashinfo,
1983	skb, doff: __tcp_hdrlen(th),
1984	saddr: &ipv6_hdr(skb)->saddr, sport: th->source,
1985	daddr: &ipv6_hdr(skb)->daddr,
1986	ntohs(th->dest),
1987	dif: tcp_v6_iif_l3_slave(skb),
1988	sdif);
1989	if (sk2) {
1990	struct inet_timewait_sock *tw = inet_twsk(sk);
1991	inet_twsk_deschedule_put(tw);
1992	sk = sk2;
1993	tcp_v6_restore_cb(skb);
1994	refcounted = false;
1995	__this_cpu_write(tcp_tw_isn, isn);
1996	goto process;
1997	}
1998	}
1999	/* to ACK */
2000	fallthrough;
2001	case TCP_TW_ACK:
2002	case TCP_TW_ACK_OOW:
2003	tcp_v6_timewait_ack(sk, skb, tw_status);
2004	break;
2005	case TCP_TW_RST:
2006	tcp_v6_send_reset(sk, skb, reason: SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
2007	inet_twsk_deschedule_put(tw: inet_twsk(sk));
2008	goto discard_it;
2009	case TCP_TW_SUCCESS:
2010	;
2011	}
2012	goto discard_it;
2013	}
2014
2015	void tcp_v6_early_demux(struct sk_buff *skb)
2016	{
2017	struct net *net = dev_net_rcu(dev: skb->dev);
2018	const struct ipv6hdr *hdr;
2019	const struct tcphdr *th;
2020	struct sock *sk;
2021
2022	if (skb->pkt_type != PACKET_HOST)
2023	return;
2024
2025	if (!pskb_may_pull(skb, len: skb_transport_offset(skb) + sizeof(struct tcphdr)))
2026	return;
2027
2028	hdr = ipv6_hdr(skb);
2029	th = tcp_hdr(skb);
2030
2031	if (th->doff < sizeof(struct tcphdr) / 4)
2032	return;
2033
2034	/* Note : We use inet6_iif() here, not tcp_v6_iif() */
2035	sk = __inet6_lookup_established(net, hashinfo: net->ipv4.tcp_death_row.hashinfo,
2036	saddr: &hdr->saddr, sport: th->source,
2037	daddr: &hdr->daddr, ntohs(th->dest),
2038	dif: inet6_iif(skb), sdif: inet6_sdif(skb));
2039	if (sk) {
2040	skb->sk = sk;
2041	skb->destructor = sock_edemux;
2042	if (sk_fullsock(sk)) {
2043	struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
2044
2045	if (dst)
2046	dst = dst_check(dst, cookie: sk->sk_rx_dst_cookie);
2047	if (dst &&
2048	sk->sk_rx_dst_ifindex == skb->skb_iif)
2049	skb_dst_set_noref(skb, dst);
2050	}
2051	}
2052	}
2053
2054	static struct timewait_sock_ops tcp6_timewait_sock_ops = {
2055	.twsk_obj_size = sizeof(struct tcp6_timewait_sock),
2056	.twsk_destructor = tcp_twsk_destructor,
2057	};
2058
2059	INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
2060	{
2061	__tcp_v6_send_check(skb, saddr: &sk->sk_v6_rcv_saddr, daddr: &sk->sk_v6_daddr);
2062	}
2063
2064	const struct inet_connection_sock_af_ops ipv6_specific = {
2065	.queue_xmit = inet6_csk_xmit,
2066	.send_check = tcp_v6_send_check,
2067	.rebuild_header = inet6_sk_rebuild_header,
2068	.sk_rx_dst_set = inet6_sk_rx_dst_set,
2069	.conn_request = tcp_v6_conn_request,
2070	.syn_recv_sock = tcp_v6_syn_recv_sock,
2071	.net_header_len = sizeof(struct ipv6hdr),
2072	.setsockopt = ipv6_setsockopt,
2073	.getsockopt = ipv6_getsockopt,
2074	.mtu_reduced = tcp_v6_mtu_reduced,
2075	};
2076
2077	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2078	static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
2079	#ifdef CONFIG_TCP_MD5SIG
2080	.md5_lookup = tcp_v6_md5_lookup,
2081	.calc_md5_hash = tcp_v6_md5_hash_skb,
2082	.md5_parse = tcp_v6_parse_md5_keys,
2083	#endif
2084	#ifdef CONFIG_TCP_AO
2085	.ao_lookup = tcp_v6_ao_lookup,
2086	.calc_ao_hash = tcp_v6_ao_hash_skb,
2087	.ao_parse = tcp_v6_parse_ao,
2088	.ao_calc_key_sk = tcp_v6_ao_calc_key_sk,
2089	#endif
2090	};
2091	#endif
2092
2093	/*
2094	* TCP over IPv4 via INET6 API
2095	*/
2096	static const struct inet_connection_sock_af_ops ipv6_mapped = {
2097	.queue_xmit = ip_queue_xmit,
2098	.send_check = tcp_v4_send_check,
2099	.rebuild_header = inet_sk_rebuild_header,
2100	.sk_rx_dst_set = inet_sk_rx_dst_set,
2101	.conn_request = tcp_v6_conn_request,
2102	.syn_recv_sock = tcp_v6_syn_recv_sock,
2103	.net_header_len = sizeof(struct iphdr),
2104	.setsockopt = ipv6_setsockopt,
2105	.getsockopt = ipv6_getsockopt,
2106	.mtu_reduced = tcp_v4_mtu_reduced,
2107	};
2108
2109	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2110	static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
2111	#ifdef CONFIG_TCP_MD5SIG
2112	.md5_lookup = tcp_v4_md5_lookup,
2113	.calc_md5_hash = tcp_v4_md5_hash_skb,
2114	.md5_parse = tcp_v6_parse_md5_keys,
2115	#endif
2116	#ifdef CONFIG_TCP_AO
2117	.ao_lookup = tcp_v6_ao_lookup,
2118	.calc_ao_hash = tcp_v4_ao_hash_skb,
2119	.ao_parse = tcp_v6_parse_ao,
2120	.ao_calc_key_sk = tcp_v4_ao_calc_key_sk,
2121	#endif
2122	};
2123	#endif
2124
2125	/* NOTE: A lot of things set to zero explicitly by call to
2126	* sk_alloc() so need not be done here.
2127	*/
2128	static int tcp_v6_init_sock(struct sock *sk)
2129	{
2130	struct inet_connection_sock *icsk = inet_csk(sk);
2131
2132	tcp_init_sock(sk);
2133
2134	icsk->icsk_af_ops = &ipv6_specific;
2135
2136	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2137	tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
2138	#endif
2139
2140	return 0;
2141	}
2142
2143	#ifdef CONFIG_PROC_FS
2144	/* Proc filesystem TCPv6 sock list dumping. */
2145	static void get_openreq6(struct seq_file *seq,
2146	const struct request_sock *req, int i)
2147	{
2148	long ttd = req->rsk_timer.expires - jiffies;
2149	const struct in6_addr *src = &inet_rsk(sk: req)->ir_v6_loc_addr;
2150	const struct in6_addr *dest = &inet_rsk(sk: req)->ir_v6_rmt_addr;
2151
2152	if (ttd < 0)
2153	ttd = 0;
2154
2155	seq_printf(m: seq,
2156	fmt: "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2157	"%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
2158	i,
2159	src->s6_addr32[0], src->s6_addr32[1],
2160	src->s6_addr32[2], src->s6_addr32[3],
2161	inet_rsk(sk: req)->ir_num,
2162	dest->s6_addr32[0], dest->s6_addr32[1],
2163	dest->s6_addr32[2], dest->s6_addr32[3],
2164	ntohs(inet_rsk(req)->ir_rmt_port),
2165	TCP_SYN_RECV,
2166	0, 0, /* could print option size, but that is af dependent. */
2167	1, /* timers active (only the expire timer) */
2168	jiffies_to_clock_t(x: ttd),
2169	req->num_timeout,
2170	from_kuid_munged(to: seq_user_ns(seq),
2171	uid: sock_i_uid(sk: req->rsk_listener)),
2172	0, /* non standard timer */
2173	0, /* open_requests have no inode */
2174	0, req);
2175	}
2176
2177	static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
2178	{
2179	const struct in6_addr *dest, *src;
2180	__u16 destp, srcp;
2181	int timer_active;
2182	unsigned long timer_expires;
2183	const struct inet_sock *inet = inet_sk(sp);
2184	const struct tcp_sock *tp = tcp_sk(sp);
2185	const struct inet_connection_sock *icsk = inet_csk(sp);
2186	const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2187	u8 icsk_pending;
2188	int rx_queue;
2189	int state;
2190
2191	dest = &sp->sk_v6_daddr;
2192	src = &sp->sk_v6_rcv_saddr;
2193	destp = ntohs(inet->inet_dport);
2194	srcp = ntohs(inet->inet_sport);
2195
2196	icsk_pending = smp_load_acquire(&icsk->icsk_pending);
2197	if (icsk_pending == ICSK_TIME_RETRANS ||
2198	icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2199	icsk_pending == ICSK_TIME_LOSS_PROBE) {
2200	timer_active = 1;
2201	timer_expires = icsk_timeout(icsk);
2202	} else if (icsk_pending == ICSK_TIME_PROBE0) {
2203	timer_active = 4;
2204	timer_expires = icsk_timeout(icsk);
2205	} else if (timer_pending(timer: &sp->sk_timer)) {
2206	timer_active = 2;
2207	timer_expires = sp->sk_timer.expires;
2208	} else {
2209	timer_active = 0;
2210	timer_expires = jiffies;
2211	}
2212
2213	state = inet_sk_state_load(sk: sp);
2214	if (state == TCP_LISTEN)
2215	rx_queue = READ_ONCE(sp->sk_ack_backlog);
2216	else
2217	/* Because we don't lock the socket,
2218	* we might find a transient negative value.
2219	*/
2220	rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2221	READ_ONCE(tp->copied_seq), 0);
2222
2223	seq_printf(m: seq,
2224	fmt: "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2225	"%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
2226	i,
2227	src->s6_addr32[0], src->s6_addr32[1],
2228	src->s6_addr32[2], src->s6_addr32[3], srcp,
2229	dest->s6_addr32[0], dest->s6_addr32[1],
2230	dest->s6_addr32[2], dest->s6_addr32[3], destp,
2231	state,
2232	READ_ONCE(tp->write_seq) - tp->snd_una,
2233	rx_queue,
2234	timer_active,
2235	jiffies_delta_to_clock_t(delta: timer_expires - jiffies),
2236	icsk->icsk_retransmits,
2237	from_kuid_munged(to: seq_user_ns(seq), uid: sock_i_uid(sk: sp)),
2238	icsk->icsk_probes_out,
2239	sock_i_ino(sk: sp),
2240	refcount_read(r: &sp->sk_refcnt), sp,
2241	jiffies_to_clock_t(x: icsk->icsk_rto),
2242	jiffies_to_clock_t(x: icsk->icsk_ack.ato),
2243	(icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk: sp),
2244	tcp_snd_cwnd(tp),
2245	state == TCP_LISTEN ?
2246	fastopenq->max_qlen :
2247	(tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2248	);
2249	}
2250
2251	static void get_timewait6_sock(struct seq_file *seq,
2252	struct inet_timewait_sock *tw, int i)
2253	{
2254	long delta = tw->tw_timer.expires - jiffies;
2255	const struct in6_addr *dest, *src;
2256	__u16 destp, srcp;
2257
2258	dest = &tw->tw_v6_daddr;
2259	src = &tw->tw_v6_rcv_saddr;
2260	destp = ntohs(tw->tw_dport);
2261	srcp = ntohs(tw->tw_sport);
2262
2263	seq_printf(m: seq,
2264	fmt: "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2265	"%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2266	i,
2267	src->s6_addr32[0], src->s6_addr32[1],
2268	src->s6_addr32[2], src->s6_addr32[3], srcp,
2269	dest->s6_addr32[0], dest->s6_addr32[1],
2270	dest->s6_addr32[2], dest->s6_addr32[3], destp,
2271	READ_ONCE(tw->tw_substate), 0, 0,
2272	3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2273	refcount_read(r: &tw->tw_refcnt), tw);
2274	}
2275
2276	static int tcp6_seq_show(struct seq_file *seq, void *v)
2277	{
2278	struct tcp_iter_state *st;
2279	struct sock *sk = v;
2280
2281	if (v == SEQ_START_TOKEN) {
2282	seq_puts(m: seq,
2283	s: " sl "
2284	"local_address "
2285	"remote_address "
2286	"st tx_queue rx_queue tr tm->when retrnsmt"
2287	" uid timeout inode\n");
2288	goto out;
2289	}
2290	st = seq->private;
2291
2292	if (sk->sk_state == TCP_TIME_WAIT)
2293	get_timewait6_sock(seq, tw: v, i: st->num);
2294	else if (sk->sk_state == TCP_NEW_SYN_RECV)
2295	get_openreq6(seq, req: v, i: st->num);
2296	else
2297	get_tcp6_sock(seq, sp: v, i: st->num);
2298	out:
2299	return 0;
2300	}
2301
2302	static const struct seq_operations tcp6_seq_ops = {
2303	.show = tcp6_seq_show,
2304	.start = tcp_seq_start,
2305	.next = tcp_seq_next,
2306	.stop = tcp_seq_stop,
2307	};
2308
2309	static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2310	.family = AF_INET6,
2311	};
2312
2313	int __net_init tcp6_proc_init(struct net *net)
2314	{
2315	if (!proc_create_net_data(name: "tcp6", mode: 0444, parent: net->proc_net, ops: &tcp6_seq_ops,
2316	state_size: sizeof(struct tcp_iter_state), data: &tcp6_seq_afinfo))
2317	return -ENOMEM;
2318	return 0;
2319	}
2320
2321	void tcp6_proc_exit(struct net *net)
2322	{
2323	remove_proc_entry("tcp6", net->proc_net);
2324	}
2325	#endif
2326
2327	struct proto tcpv6_prot = {
2328	.name = "TCPv6",
2329	.owner = THIS_MODULE,
2330	.close = tcp_close,
2331	.pre_connect = tcp_v6_pre_connect,
2332	.connect = tcp_v6_connect,
2333	.disconnect = tcp_disconnect,
2334	.accept = inet_csk_accept,
2335	.ioctl = tcp_ioctl,
2336	.init = tcp_v6_init_sock,
2337	.destroy = tcp_v4_destroy_sock,
2338	.shutdown = tcp_shutdown,
2339	.setsockopt = tcp_setsockopt,
2340	.getsockopt = tcp_getsockopt,
2341	.bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
2342	.keepalive = tcp_set_keepalive,
2343	.recvmsg = tcp_recvmsg,
2344	.sendmsg = tcp_sendmsg,
2345	.splice_eof = tcp_splice_eof,
2346	.backlog_rcv = tcp_v6_do_rcv,
2347	.release_cb = tcp_release_cb,
2348	.hash = inet6_hash,
2349	.unhash = inet_unhash,
2350	.get_port = inet_csk_get_port,
2351	.put_port = inet_put_port,
2352	#ifdef CONFIG_BPF_SYSCALL
2353	.psock_update_sk_prot = tcp_bpf_update_proto,
2354	#endif
2355	.enter_memory_pressure = tcp_enter_memory_pressure,
2356	.leave_memory_pressure = tcp_leave_memory_pressure,
2357	.stream_memory_free = tcp_stream_memory_free,
2358	.sockets_allocated = &tcp_sockets_allocated,
2359
2360	.memory_allocated = &tcp_memory_allocated,
2361	.per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc,
2362
2363	.memory_pressure = &tcp_memory_pressure,
2364	.orphan_count = &tcp_orphan_count,
2365	.sysctl_mem = sysctl_tcp_mem,
2366	.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2367	.sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2368	.max_header = MAX_TCP_HEADER,
2369	.obj_size = sizeof(struct tcp6_sock),
2370	.ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
2371	.slab_flags = SLAB_TYPESAFE_BY_RCU,
2372	.twsk_prot = &tcp6_timewait_sock_ops,
2373	.rsk_prot = &tcp6_request_sock_ops,
2374	.h.hashinfo = NULL,
2375	.no_autobind = true,
2376	.diag_destroy = tcp_abort,
2377	};
2378	EXPORT_SYMBOL_GPL(tcpv6_prot);
2379
2380
2381	static struct inet_protosw tcpv6_protosw = {
2382	.type = SOCK_STREAM,
2383	.protocol = IPPROTO_TCP,
2384	.prot = &tcpv6_prot,
2385	.ops = &inet6_stream_ops,
2386	.flags = INET_PROTOSW_PERMANENT |
2387	INET_PROTOSW_ICSK,
2388	};
2389
2390	static int __net_init tcpv6_net_init(struct net *net)
2391	{
2392	int res;
2393
2394	res = inet_ctl_sock_create(sk: &net->ipv6.tcp_sk, PF_INET6,
2395	type: SOCK_RAW, IPPROTO_TCP, net);
2396	if (!res)
2397	net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC;
2398
2399	return res;
2400	}
2401
2402	static void __net_exit tcpv6_net_exit(struct net *net)
2403	{
2404	inet_ctl_sock_destroy(sk: net->ipv6.tcp_sk);
2405	}
2406
2407	static struct pernet_operations tcpv6_net_ops = {
2408	.init = tcpv6_net_init,
2409	.exit = tcpv6_net_exit,
2410	};
2411
2412	int __init tcpv6_init(void)
2413	{
2414	int ret;
2415
2416	net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
2417	.handler = tcp_v6_rcv,
2418	.err_handler = tcp_v6_err,
2419	.flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
2420	};
2421	ret = inet6_add_protocol(prot: &net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2422	if (ret)
2423	goto out;
2424
2425	/* register inet6 protocol */
2426	ret = inet6_register_protosw(p: &tcpv6_protosw);
2427	if (ret)
2428	goto out_tcpv6_protocol;
2429
2430	ret = register_pernet_subsys(&tcpv6_net_ops);
2431	if (ret)
2432	goto out_tcpv6_protosw;
2433
2434	ret = mptcpv6_init();
2435	if (ret)
2436	goto out_tcpv6_pernet_subsys;
2437
2438	out:
2439	return ret;
2440
2441	out_tcpv6_pernet_subsys:
2442	unregister_pernet_subsys(&tcpv6_net_ops);
2443	out_tcpv6_protosw:
2444	inet6_unregister_protosw(p: &tcpv6_protosw);
2445	out_tcpv6_protocol:
2446	inet6_del_protocol(prot: &net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2447	goto out;
2448	}
2449
2450	void tcpv6_exit(void)
2451	{
2452	unregister_pernet_subsys(&tcpv6_net_ops);
2453	inet6_unregister_protosw(p: &tcpv6_protosw);
2454	inet6_del_protocol(prot: &net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2455	}
2456