1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* TCP over IPv6
4	* Linux INET6 implementation
5	*
6	* Authors:
7	* Pedro Roque <roque@di.fc.ul.pt>
8	*
9	* Based on:
10	* linux/net/ipv4/tcp.c
11	* linux/net/ipv4/tcp_input.c
12	* linux/net/ipv4/tcp_output.c
13	*
14	* Fixes:
15	* Hideaki YOSHIFUJI : sin6_scope_id support
16	* YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
17	* Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
18	* a single port at the same time.
19	* YOSHIFUJI Hideaki @USAGI: convert /proc/net/tcp6 to seq_file.
20	*/
21
22	#include <linux/bottom_half.h>
23	#include <linux/module.h>
24	#include <linux/errno.h>
25	#include <linux/types.h>
26	#include <linux/socket.h>
27	#include <linux/sockios.h>
28	#include <linux/net.h>
29	#include <linux/jiffies.h>
30	#include <linux/in.h>
31	#include <linux/in6.h>
32	#include <linux/netdevice.h>
33	#include <linux/init.h>
34	#include <linux/jhash.h>
35	#include <linux/ipsec.h>
36	#include <linux/times.h>
37	#include <linux/slab.h>
38	#include <linux/uaccess.h>
39	#include <linux/ipv6.h>
40	#include <linux/icmpv6.h>
41	#include <linux/random.h>
42	#include <linux/indirect_call_wrapper.h>
43
44	#include <net/tcp.h>
45	#include <net/ndisc.h>
46	#include <net/inet6_hashtables.h>
47	#include <net/inet6_connection_sock.h>
48	#include <net/ipv6.h>
49	#include <net/transp_v6.h>
50	#include <net/addrconf.h>
51	#include <net/ip6_route.h>
52	#include <net/ip6_checksum.h>
53	#include <net/inet_ecn.h>
54	#include <net/protocol.h>
55	#include <net/xfrm.h>
56	#include <net/snmp.h>
57	#include <net/dsfield.h>
58	#include <net/timewait_sock.h>
59	#include <net/inet_common.h>
60	#include <net/secure_seq.h>
61	#include <net/busy_poll.h>
62
63	#include <linux/proc_fs.h>
64	#include <linux/seq_file.h>
65
66	#include <crypto/hash.h>
67	#include <linux/scatterlist.h>
68
69	#include <trace/events/tcp.h>
70
71	static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb);
72	static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
73	struct request_sock *req);
74
75	INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
76
77	static const struct inet_connection_sock_af_ops ipv6_mapped;
78	const struct inet_connection_sock_af_ops ipv6_specific;
79	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
80	static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
81	static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
82	#endif
83
84	/* Helper returning the inet6 address from a given tcp socket.
85	* It can be used in TCP stack instead of inet6_sk(sk).
86	* This avoids a dereference and allow compiler optimizations.
87	* It is a specialized version of inet6_sk_generic().
88	*/
89	#define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
90	struct tcp6_sock, tcp)->inet6)
91
92	static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
93	{
94	struct dst_entry *dst = skb_dst(skb);
95
96	if (dst && dst_hold_safe(dst)) {
97	const struct rt6_info *rt = (const struct rt6_info *)dst;
98
99	rcu_assign_pointer(sk->sk_rx_dst, dst);
100	sk->sk_rx_dst_ifindex = skb->skb_iif;
101	sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
102	}
103	}
104
105	static u32 tcp_v6_init_seq(const struct sk_buff *skb)
106	{
107	return secure_tcpv6_seq(saddr: ipv6_hdr(skb)->daddr.s6_addr32,
108	daddr: ipv6_hdr(skb)->saddr.s6_addr32,
109	sport: tcp_hdr(skb)->dest,
110	dport: tcp_hdr(skb)->source);
111	}
112
113	static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb)
114	{
115	return secure_tcpv6_ts_off(net, saddr: ipv6_hdr(skb)->daddr.s6_addr32,
116	daddr: ipv6_hdr(skb)->saddr.s6_addr32);
117	}
118
119	static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
120	int addr_len)
121	{
122	/* This check is replicated from tcp_v6_connect() and intended to
123	* prevent BPF program called below from accessing bytes that are out
124	* of the bound specified by user in addr_len.
125	*/
126	if (addr_len < SIN6_LEN_RFC2133)
127	return -EINVAL;
128
129	sock_owned_by_me(sk);
130
131	return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
132	}
133
134	static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
135	int addr_len)
136	{
137	struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
138	struct inet_connection_sock *icsk = inet_csk(sk);
139	struct in6_addr *saddr = NULL, *final_p, final;
140	struct inet_timewait_death_row *tcp_death_row;
141	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
142	struct inet_sock *inet = inet_sk(sk);
143	struct tcp_sock *tp = tcp_sk(sk);
144	struct net *net = sock_net(sk);
145	struct ipv6_txoptions *opt;
146	struct dst_entry *dst;
147	struct flowi6 fl6;
148	int addr_type;
149	int err;
150
151	if (addr_len < SIN6_LEN_RFC2133)
152	return -EINVAL;
153
154	if (usin->sin6_family != AF_INET6)
155	return -EAFNOSUPPORT;
156
157	memset(&fl6, 0, sizeof(fl6));
158
159	if (inet6_test_bit(SNDFLOW, sk)) {
160	fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
161	IP6_ECN_flow_init(fl6.flowlabel);
162	if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
163	struct ip6_flowlabel *flowlabel;
164	flowlabel = fl6_sock_lookup(sk, label: fl6.flowlabel);
165	if (IS_ERR(ptr: flowlabel))
166	return -EINVAL;
167	fl6_sock_release(fl: flowlabel);
168	}
169	}
170
171	/*
172	* connect() to INADDR_ANY means loopback (BSD'ism).
173	*/
174
175	if (ipv6_addr_any(a: &usin->sin6_addr)) {
176	if (ipv6_addr_v4mapped(a: &sk->sk_v6_rcv_saddr))
177	ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
178	v4mapped: &usin->sin6_addr);
179	else
180	usin->sin6_addr = in6addr_loopback;
181	}
182
183	addr_type = ipv6_addr_type(addr: &usin->sin6_addr);
184
185	if (addr_type & IPV6_ADDR_MULTICAST)
186	return -ENETUNREACH;
187
188	if (addr_type&IPV6_ADDR_LINKLOCAL) {
189	if (addr_len >= sizeof(struct sockaddr_in6) &&
190	usin->sin6_scope_id) {
191	/* If interface is set while binding, indices
192	* must coincide.
193	*/
194	if (!sk_dev_equal_l3scope(sk, dif: usin->sin6_scope_id))
195	return -EINVAL;
196
197	sk->sk_bound_dev_if = usin->sin6_scope_id;
198	}
199
200	/* Connect to link-local address requires an interface */
201	if (!sk->sk_bound_dev_if)
202	return -EINVAL;
203	}
204
205	if (tp->rx_opt.ts_recent_stamp &&
206	!ipv6_addr_equal(a1: &sk->sk_v6_daddr, a2: &usin->sin6_addr)) {
207	tp->rx_opt.ts_recent = 0;
208	tp->rx_opt.ts_recent_stamp = 0;
209	WRITE_ONCE(tp->write_seq, 0);
210	}
211
212	sk->sk_v6_daddr = usin->sin6_addr;
213	np->flow_label = fl6.flowlabel;
214
215	/*
216	* TCP over IPv4
217	*/
218
219	if (addr_type & IPV6_ADDR_MAPPED) {
220	u32 exthdrlen = icsk->icsk_ext_hdr_len;
221	struct sockaddr_in sin;
222
223	if (ipv6_only_sock(sk))
224	return -ENETUNREACH;
225
226	sin.sin_family = AF_INET;
227	sin.sin_port = usin->sin6_port;
228	sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
229
230	/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
231	WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
232	if (sk_is_mptcp(sk))
233	mptcpv6_handle_mapped(sk, mapped: true);
234	sk->sk_backlog_rcv = tcp_v4_do_rcv;
235	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
236	tp->af_specific = &tcp_sock_ipv6_mapped_specific;
237	#endif
238
239	err = tcp_v4_connect(sk, uaddr: (struct sockaddr *)&sin, addr_len: sizeof(sin));
240
241	if (err) {
242	icsk->icsk_ext_hdr_len = exthdrlen;
243	/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
244	WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
245	if (sk_is_mptcp(sk))
246	mptcpv6_handle_mapped(sk, mapped: false);
247	sk->sk_backlog_rcv = tcp_v6_do_rcv;
248	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
249	tp->af_specific = &tcp_sock_ipv6_specific;
250	#endif
251	goto failure;
252	}
253	np->saddr = sk->sk_v6_rcv_saddr;
254
255	return err;
256	}
257
258	if (!ipv6_addr_any(a: &sk->sk_v6_rcv_saddr))
259	saddr = &sk->sk_v6_rcv_saddr;
260
261	fl6.flowi6_proto = IPPROTO_TCP;
262	fl6.daddr = sk->sk_v6_daddr;
263	fl6.saddr = saddr ? *saddr : np->saddr;
264	fl6.flowlabel = ip6_make_flowinfo(tclass: np->tclass, flowlabel: np->flow_label);
265	fl6.flowi6_oif = sk->sk_bound_dev_if;
266	fl6.flowi6_mark = sk->sk_mark;
267	fl6.fl6_dport = usin->sin6_port;
268	fl6.fl6_sport = inet->inet_sport;
269	fl6.flowi6_uid = sk->sk_uid;
270
271	opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
272	final_p = fl6_update_dst(fl6: &fl6, opt, orig: &final);
273
274	security_sk_classify_flow(sk, flic: flowi6_to_flowi_common(fl6: &fl6));
275
276	dst = ip6_dst_lookup_flow(net, sk, fl6: &fl6, final_dst: final_p);
277	if (IS_ERR(ptr: dst)) {
278	err = PTR_ERR(ptr: dst);
279	goto failure;
280	}
281
282	tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
283	tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
284
285	if (!saddr) {
286	saddr = &fl6.saddr;
287
288	err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
289	if (err)
290	goto failure;
291	}
292
293	/* set the source address */
294	np->saddr = *saddr;
295	inet->inet_rcv_saddr = LOOPBACK4_IPV6;
296
297	sk->sk_gso_type = SKB_GSO_TCPV6;
298	ip6_dst_store(sk, dst, NULL, NULL);
299
300	icsk->icsk_ext_hdr_len = 0;
301	if (opt)
302	icsk->icsk_ext_hdr_len = opt->opt_flen +
303	opt->opt_nflen;
304
305	tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
306
307	inet->inet_dport = usin->sin6_port;
308
309	tcp_set_state(sk, state: TCP_SYN_SENT);
310	err = inet6_hash_connect(death_row: tcp_death_row, sk);
311	if (err)
312	goto late_failure;
313
314	sk_set_txhash(sk);
315
316	if (likely(!tp->repair)) {
317	if (!tp->write_seq)
318	WRITE_ONCE(tp->write_seq,
319	secure_tcpv6_seq(np->saddr.s6_addr32,
320	sk->sk_v6_daddr.s6_addr32,
321	inet->inet_sport,
322	inet->inet_dport));
323	tp->tsoffset = secure_tcpv6_ts_off(net, saddr: np->saddr.s6_addr32,
324	daddr: sk->sk_v6_daddr.s6_addr32);
325	}
326
327	if (tcp_fastopen_defer_connect(sk, err: &err))
328	return err;
329	if (err)
330	goto late_failure;
331
332	err = tcp_connect(sk);
333	if (err)
334	goto late_failure;
335
336	return 0;
337
338	late_failure:
339	tcp_set_state(sk, state: TCP_CLOSE);
340	inet_bhash2_reset_saddr(sk);
341	failure:
342	inet->inet_dport = 0;
343	sk->sk_route_caps = 0;
344	return err;
345	}
346
347	static void tcp_v6_mtu_reduced(struct sock *sk)
348	{
349	struct dst_entry *dst;
350	u32 mtu;
351
352	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
353	return;
354
355	mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
356
357	/* Drop requests trying to increase our current mss.
358	* Check done in __ip6_rt_update_pmtu() is too late.
359	*/
360	if (tcp_mtu_to_mss(sk, pmtu: mtu) >= tcp_sk(sk)->mss_cache)
361	return;
362
363	dst = inet6_csk_update_pmtu(sk, mtu);
364	if (!dst)
365	return;
366
367	if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
368	tcp_sync_mss(sk, pmtu: dst_mtu(dst));
369	tcp_simple_retransmit(sk);
370	}
371	}
372
373	static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
374	u8 type, u8 code, int offset, __be32 info)
375	{
376	const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
377	const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
378	struct net *net = dev_net(dev: skb->dev);
379	struct request_sock *fastopen;
380	struct ipv6_pinfo *np;
381	struct tcp_sock *tp;
382	__u32 seq, snd_una;
383	struct sock *sk;
384	bool fatal;
385	int err;
386
387	sk = __inet6_lookup_established(net, hashinfo: net->ipv4.tcp_death_row.hashinfo,
388	saddr: &hdr->daddr, sport: th->dest,
389	daddr: &hdr->saddr, ntohs(th->source),
390	dif: skb->dev->ifindex, sdif: inet6_sdif(skb));
391
392	if (!sk) {
393	__ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
394	ICMP6_MIB_INERRORS);
395	return -ENOENT;
396	}
397
398	if (sk->sk_state == TCP_TIME_WAIT) {
399	/* To increase the counter of ignored icmps for TCP-AO */
400	tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
401	inet_twsk_put(tw: inet_twsk(sk));
402	return 0;
403	}
404	seq = ntohl(th->seq);
405	fatal = icmpv6_err_convert(type, code, err: &err);
406	if (sk->sk_state == TCP_NEW_SYN_RECV) {
407	tcp_req_err(sk, seq, abort: fatal);
408	return 0;
409	}
410
411	if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
412	sock_put(sk);
413	return 0;
414	}
415
416	bh_lock_sock(sk);
417	if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
418	__NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
419
420	if (sk->sk_state == TCP_CLOSE)
421	goto out;
422
423	if (static_branch_unlikely(&ip6_min_hopcount)) {
424	/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
425	if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
426	__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
427	goto out;
428	}
429	}
430
431	tp = tcp_sk(sk);
432	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
433	fastopen = rcu_dereference(tp->fastopen_rsk);
434	snd_una = fastopen ? tcp_rsk(req: fastopen)->snt_isn : tp->snd_una;
435	if (sk->sk_state != TCP_LISTEN &&
436	!between(seq1: seq, seq2: snd_una, seq3: tp->snd_nxt)) {
437	__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
438	goto out;
439	}
440
441	np = tcp_inet6_sk(sk);
442
443	if (type == NDISC_REDIRECT) {
444	if (!sock_owned_by_user(sk)) {
445	struct dst_entry *dst = __sk_dst_check(sk, cookie: np->dst_cookie);
446
447	if (dst)
448	dst->ops->redirect(dst, sk, skb);
449	}
450	goto out;
451	}
452
453	if (type == ICMPV6_PKT_TOOBIG) {
454	u32 mtu = ntohl(info);
455
456	/* We are not interested in TCP_LISTEN and open_requests
457	* (SYN-ACKs send out by Linux are always <576bytes so
458	* they should go through unfragmented).
459	*/
460	if (sk->sk_state == TCP_LISTEN)
461	goto out;
462
463	if (!ip6_sk_accept_pmtu(sk))
464	goto out;
465
466	if (mtu < IPV6_MIN_MTU)
467	goto out;
468
469	WRITE_ONCE(tp->mtu_info, mtu);
470
471	if (!sock_owned_by_user(sk))
472	tcp_v6_mtu_reduced(sk);
473	else if (!test_and_set_bit(nr: TCP_MTU_REDUCED_DEFERRED,
474	addr: &sk->sk_tsq_flags))
475	sock_hold(sk);
476	goto out;
477	}
478
479
480	/* Might be for an request_sock */
481	switch (sk->sk_state) {
482	case TCP_SYN_SENT:
483	case TCP_SYN_RECV:
484	/* Only in fast or simultaneous open. If a fast open socket is
485	* already accepted it is treated as a connected one below.
486	*/
487	if (fastopen && !fastopen->sk)
488	break;
489
490	ipv6_icmp_error(sk, skb, err, port: th->dest, ntohl(info), payload: (u8 *)th);
491
492	if (!sock_owned_by_user(sk)) {
493	WRITE_ONCE(sk->sk_err, err);
494	sk_error_report(sk); /* Wake people up to see the error (see connect in sock.c) */
495
496	tcp_done(sk);
497	} else {
498	WRITE_ONCE(sk->sk_err_soft, err);
499	}
500	goto out;
501	case TCP_LISTEN:
502	break;
503	default:
504	/* check if this ICMP message allows revert of backoff.
505	* (see RFC 6069)
506	*/
507	if (!fastopen && type == ICMPV6_DEST_UNREACH &&
508	code == ICMPV6_NOROUTE)
509	tcp_ld_RTO_revert(sk, seq);
510	}
511
512	if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
513	WRITE_ONCE(sk->sk_err, err);
514	sk_error_report(sk);
515	} else {
516	WRITE_ONCE(sk->sk_err_soft, err);
517	}
518	out:
519	bh_unlock_sock(sk);
520	sock_put(sk);
521	return 0;
522	}
523
524
525	static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
526	struct flowi *fl,
527	struct request_sock *req,
528	struct tcp_fastopen_cookie *foc,
529	enum tcp_synack_type synack_type,
530	struct sk_buff *syn_skb)
531	{
532	struct inet_request_sock *ireq = inet_rsk(sk: req);
533	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
534	struct ipv6_txoptions *opt;
535	struct flowi6 *fl6 = &fl->u.ip6;
536	struct sk_buff *skb;
537	int err = -ENOMEM;
538	u8 tclass;
539
540	/* First, grab a route. */
541	if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
542	IPPROTO_TCP)) == NULL)
543	goto done;
544
545	skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
546
547	if (skb) {
548	__tcp_v6_send_check(skb, saddr: &ireq->ir_v6_loc_addr,
549	daddr: &ireq->ir_v6_rmt_addr);
550
551	fl6->daddr = ireq->ir_v6_rmt_addr;
552	if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
553	fl6->flowlabel = ip6_flowlabel(hdr: ipv6_hdr(skb: ireq->pktopts));
554
555	tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
556	(tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
557	(np->tclass & INET_ECN_MASK) :
558	np->tclass;
559
560	if (!INET_ECN_is_capable(dsfield: tclass) &&
561	tcp_bpf_ca_needs_ecn(sk: (struct sock *)req))
562	tclass |= INET_ECN_ECT_0;
563
564	rcu_read_lock();
565	opt = ireq->ipv6_opt;
566	if (!opt)
567	opt = rcu_dereference(np->opt);
568	err = ip6_xmit(sk, skb, fl6, mark: skb->mark ? : READ_ONCE(sk->sk_mark),
569	opt, tclass, READ_ONCE(sk->sk_priority));
570	rcu_read_unlock();
571	err = net_xmit_eval(err);
572	}
573
574	done:
575	return err;
576	}
577
578
579	static void tcp_v6_reqsk_destructor(struct request_sock *req)
580	{
581	kfree(objp: inet_rsk(sk: req)->ipv6_opt);
582	consume_skb(skb: inet_rsk(sk: req)->pktopts);
583	}
584
585	#ifdef CONFIG_TCP_MD5SIG
586	static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
587	const struct in6_addr *addr,
588	int l3index)
589	{
590	return tcp_md5_do_lookup(sk, l3index,
591	addr: (union tcp_md5_addr *)addr, AF_INET6);
592	}
593
594	static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
595	const struct sock *addr_sk)
596	{
597	int l3index;
598
599	l3index = l3mdev_master_ifindex_by_index(net: sock_net(sk),
600	ifindex: addr_sk->sk_bound_dev_if);
601	return tcp_v6_md5_do_lookup(sk, addr: &addr_sk->sk_v6_daddr,
602	l3index);
603	}
604
605	static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
606	sockptr_t optval, int optlen)
607	{
608	struct tcp_md5sig cmd;
609	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
610	union tcp_ao_addr *addr;
611	int l3index = 0;
612	u8 prefixlen;
613	bool l3flag;
614	u8 flags;
615
616	if (optlen < sizeof(cmd))
617	return -EINVAL;
618
619	if (copy_from_sockptr(dst: &cmd, src: optval, size: sizeof(cmd)))
620	return -EFAULT;
621
622	if (sin6->sin6_family != AF_INET6)
623	return -EINVAL;
624
625	flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
626	l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
627
628	if (optname == TCP_MD5SIG_EXT &&
629	cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
630	prefixlen = cmd.tcpm_prefixlen;
631	if (prefixlen > 128 || (ipv6_addr_v4mapped(a: &sin6->sin6_addr) &&
632	prefixlen > 32))
633	return -EINVAL;
634	} else {
635	prefixlen = ipv6_addr_v4mapped(a: &sin6->sin6_addr) ? 32 : 128;
636	}
637
638	if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
639	cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
640	struct net_device *dev;
641
642	rcu_read_lock();
643	dev = dev_get_by_index_rcu(net: sock_net(sk), ifindex: cmd.tcpm_ifindex);
644	if (dev && netif_is_l3_master(dev))
645	l3index = dev->ifindex;
646	rcu_read_unlock();
647
648	/* ok to reference set/not set outside of rcu;
649	* right now device MUST be an L3 master
650	*/
651	if (!dev || !l3index)
652	return -EINVAL;
653	}
654
655	if (!cmd.tcpm_keylen) {
656	if (ipv6_addr_v4mapped(a: &sin6->sin6_addr))
657	return tcp_md5_do_del(sk, addr: (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
658	AF_INET, prefixlen,
659	l3index, flags);
660	return tcp_md5_do_del(sk, addr: (union tcp_md5_addr *)&sin6->sin6_addr,
661	AF_INET6, prefixlen, l3index, flags);
662	}
663
664	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
665	return -EINVAL;
666
667	if (ipv6_addr_v4mapped(a: &sin6->sin6_addr)) {
668	addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];
669
670	/* Don't allow keys for peers that have a matching TCP-AO key.
671	* See the comment in tcp_ao_add_cmd()
672	*/
673	if (tcp_ao_required(sk, saddr: addr, AF_INET,
674	l3index: l3flag ? l3index : -1, stat_inc: false))
675	return -EKEYREJECTED;
676	return tcp_md5_do_add(sk, addr,
677	AF_INET, prefixlen, l3index, flags,
678	newkey: cmd.tcpm_key, newkeylen: cmd.tcpm_keylen);
679	}
680
681	addr = (union tcp_md5_addr *)&sin6->sin6_addr;
682
683	/* Don't allow keys for peers that have a matching TCP-AO key.
684	* See the comment in tcp_ao_add_cmd()
685	*/
686	if (tcp_ao_required(sk, saddr: addr, AF_INET6, l3index: l3flag ? l3index : -1, stat_inc: false))
687	return -EKEYREJECTED;
688
689	return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
690	newkey: cmd.tcpm_key, newkeylen: cmd.tcpm_keylen);
691	}
692
693	static int tcp_v6_md5_hash_headers(struct tcp_sigpool *hp,
694	const struct in6_addr *daddr,
695	const struct in6_addr *saddr,
696	const struct tcphdr *th, int nbytes)
697	{
698	struct tcp6_pseudohdr *bp;
699	struct scatterlist sg;
700	struct tcphdr *_th;
701
702	bp = hp->scratch;
703	/* 1. TCP pseudo-header (RFC2460) */
704	bp->saddr = *saddr;
705	bp->daddr = *daddr;
706	bp->protocol = cpu_to_be32(IPPROTO_TCP);
707	bp->len = cpu_to_be32(nbytes);
708
709	_th = (struct tcphdr *)(bp + 1);
710	memcpy(_th, th, sizeof(*th));
711	_th->check = 0;
712
713	sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
714	ahash_request_set_crypt(req: hp->req, src: &sg, NULL,
715	nbytes: sizeof(*bp) + sizeof(*th));
716	return crypto_ahash_update(req: hp->req);
717	}
718
719	static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
720	const struct in6_addr *daddr, struct in6_addr *saddr,
721	const struct tcphdr *th)
722	{
723	struct tcp_sigpool hp;
724
725	if (tcp_sigpool_start(id: tcp_md5_sigpool_id, c: &hp))
726	goto clear_hash_nostart;
727
728	if (crypto_ahash_init(req: hp.req))
729	goto clear_hash;
730	if (tcp_v6_md5_hash_headers(hp: &hp, daddr, saddr, th, nbytes: th->doff << 2))
731	goto clear_hash;
732	if (tcp_md5_hash_key(hp: &hp, key))
733	goto clear_hash;
734	ahash_request_set_crypt(req: hp.req, NULL, result: md5_hash, nbytes: 0);
735	if (crypto_ahash_final(req: hp.req))
736	goto clear_hash;
737
738	tcp_sigpool_end(c: &hp);
739	return 0;
740
741	clear_hash:
742	tcp_sigpool_end(c: &hp);
743	clear_hash_nostart:
744	memset(md5_hash, 0, 16);
745	return 1;
746	}
747
748	static int tcp_v6_md5_hash_skb(char *md5_hash,
749	const struct tcp_md5sig_key *key,
750	const struct sock *sk,
751	const struct sk_buff *skb)
752	{
753	const struct tcphdr *th = tcp_hdr(skb);
754	const struct in6_addr *saddr, *daddr;
755	struct tcp_sigpool hp;
756
757	if (sk) { /* valid for establish/request sockets */
758	saddr = &sk->sk_v6_rcv_saddr;
759	daddr = &sk->sk_v6_daddr;
760	} else {
761	const struct ipv6hdr *ip6h = ipv6_hdr(skb);
762	saddr = &ip6h->saddr;
763	daddr = &ip6h->daddr;
764	}
765
766	if (tcp_sigpool_start(id: tcp_md5_sigpool_id, c: &hp))
767	goto clear_hash_nostart;
768
769	if (crypto_ahash_init(req: hp.req))
770	goto clear_hash;
771
772	if (tcp_v6_md5_hash_headers(hp: &hp, daddr, saddr, th, nbytes: skb->len))
773	goto clear_hash;
774	if (tcp_sigpool_hash_skb_data(hp: &hp, skb, header_len: th->doff << 2))
775	goto clear_hash;
776	if (tcp_md5_hash_key(hp: &hp, key))
777	goto clear_hash;
778	ahash_request_set_crypt(req: hp.req, NULL, result: md5_hash, nbytes: 0);
779	if (crypto_ahash_final(req: hp.req))
780	goto clear_hash;
781
782	tcp_sigpool_end(c: &hp);
783	return 0;
784
785	clear_hash:
786	tcp_sigpool_end(c: &hp);
787	clear_hash_nostart:
788	memset(md5_hash, 0, 16);
789	return 1;
790	}
791	#endif
792
793	static void tcp_v6_init_req(struct request_sock *req,
794	const struct sock *sk_listener,
795	struct sk_buff *skb)
796	{
797	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
798	struct inet_request_sock *ireq = inet_rsk(sk: req);
799	const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
800
801	ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
802	ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
803
804	/* So that link locals have meaning */
805	if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
806	ipv6_addr_type(addr: &ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
807	ireq->ir_iif = tcp_v6_iif(skb);
808
809	if (!TCP_SKB_CB(skb)->tcp_tw_isn &&
810	(ipv6_opt_accepted(sk: sk_listener, skb, opt: &TCP_SKB_CB(skb)->header.h6) ||
811	np->rxopt.bits.rxinfo ||
812	np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
813	np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
814	refcount_inc(r: &skb->users);
815	ireq->pktopts = skb;
816	}
817	}
818
819	static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
820	struct sk_buff *skb,
821	struct flowi *fl,
822	struct request_sock *req)
823	{
824	tcp_v6_init_req(req, sk_listener: sk, skb);
825
826	if (security_inet_conn_request(sk, skb, req))
827	return NULL;
828
829	return inet6_csk_route_req(sk, fl6: &fl->u.ip6, req, IPPROTO_TCP);
830	}
831
832	struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
833	.family = AF_INET6,
834	.obj_size = sizeof(struct tcp6_request_sock),
835	.rtx_syn_ack = tcp_rtx_synack,
836	.send_ack = tcp_v6_reqsk_send_ack,
837	.destructor = tcp_v6_reqsk_destructor,
838	.send_reset = tcp_v6_send_reset,
839	.syn_ack_timeout = tcp_syn_ack_timeout,
840	};
841
842	const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
843	.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) -
844	sizeof(struct ipv6hdr),
845	#ifdef CONFIG_TCP_MD5SIG
846	.req_md5_lookup = tcp_v6_md5_lookup,
847	.calc_md5_hash = tcp_v6_md5_hash_skb,
848	#endif
849	#ifdef CONFIG_TCP_AO
850	.ao_lookup = tcp_v6_ao_lookup_rsk,
851	.ao_calc_key = tcp_v6_ao_calc_key_rsk,
852	.ao_synack_hash = tcp_v6_ao_synack_hash,
853	#endif
854	#ifdef CONFIG_SYN_COOKIES
855	.cookie_init_seq = cookie_v6_init_sequence,
856	#endif
857	.route_req = tcp_v6_route_req,
858	.init_seq = tcp_v6_init_seq,
859	.init_ts_off = tcp_v6_init_ts_off,
860	.send_synack = tcp_v6_send_synack,
861	};
862
863	static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
864	u32 ack, u32 win, u32 tsval, u32 tsecr,
865	int oif, int rst, u8 tclass, __be32 label,
866	u32 priority, u32 txhash, struct tcp_key *key)
867	{
868	const struct tcphdr *th = tcp_hdr(skb);
869	struct tcphdr *t1;
870	struct sk_buff *buff;
871	struct flowi6 fl6;
872	struct net *net = sk ? sock_net(sk) : dev_net(dev: skb_dst(skb)->dev);
873	struct sock *ctl_sk = net->ipv6.tcp_sk;
874	unsigned int tot_len = sizeof(struct tcphdr);
875	__be32 mrst = 0, *topt;
876	struct dst_entry *dst;
877	__u32 mark = 0;
878
879	if (tsecr)
880	tot_len += TCPOLEN_TSTAMP_ALIGNED;
881	if (tcp_key_is_md5(key))
882	tot_len += TCPOLEN_MD5SIG_ALIGNED;
883	if (tcp_key_is_ao(key))
884	tot_len += tcp_ao_len(key: key->ao_key);
885
886	#ifdef CONFIG_MPTCP
887	if (rst && !tcp_key_is_md5(key)) {
888	mrst = mptcp_reset_option(skb);
889
890	if (mrst)
891	tot_len += sizeof(__be32);
892	}
893	#endif
894
895	buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
896	if (!buff)
897	return;
898
899	skb_reserve(skb: buff, MAX_TCP_HEADER);
900
901	t1 = skb_push(skb: buff, len: tot_len);
902	skb_reset_transport_header(skb: buff);
903
904	/* Swap the send and the receive. */
905	memset(t1, 0, sizeof(*t1));
906	t1->dest = th->source;
907	t1->source = th->dest;
908	t1->doff = tot_len / 4;
909	t1->seq = htonl(seq);
910	t1->ack_seq = htonl(ack);
911	t1->ack = !rst || !th->ack;
912	t1->rst = rst;
913	t1->window = htons(win);
914
915	topt = (__be32 *)(t1 + 1);
916
917	if (tsecr) {
918	*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
919	(TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
920	*topt++ = htonl(tsval);
921	*topt++ = htonl(tsecr);
922	}
923
924	if (mrst)
925	*topt++ = mrst;
926
927	#ifdef CONFIG_TCP_MD5SIG
928	if (tcp_key_is_md5(key)) {
929	*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
930	(TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
931	tcp_v6_md5_hash_hdr(md5_hash: (__u8 *)topt, key: key->md5_key,
932	daddr: &ipv6_hdr(skb)->saddr,
933	saddr: &ipv6_hdr(skb)->daddr, th: t1);
934	}
935	#endif
936	#ifdef CONFIG_TCP_AO
937	if (tcp_key_is_ao(key)) {
938	*topt++ = htonl((TCPOPT_AO << 24) |
939	(tcp_ao_len(key->ao_key) << 16) |
940	(key->ao_key->sndid << 8) |
941	(key->rcv_next));
942
943	tcp_ao_hash_hdr(AF_INET6, ao_hash: (char *)topt, key: key->ao_key,
944	tkey: key->traffic_key,
945	daddr: (union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
946	saddr: (union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
947	th: t1, sne: key->sne);
948	}
949	#endif
950
951	memset(&fl6, 0, sizeof(fl6));
952	fl6.daddr = ipv6_hdr(skb)->saddr;
953	fl6.saddr = ipv6_hdr(skb)->daddr;
954	fl6.flowlabel = label;
955
956	buff->ip_summed = CHECKSUM_PARTIAL;
957
958	__tcp_v6_send_check(skb: buff, saddr: &fl6.saddr, daddr: &fl6.daddr);
959
960	fl6.flowi6_proto = IPPROTO_TCP;
961	if (rt6_need_strict(daddr: &fl6.daddr) && !oif)
962	fl6.flowi6_oif = tcp_v6_iif(skb);
963	else {
964	if (!oif && netif_index_is_l3_master(net, ifindex: skb->skb_iif))
965	oif = skb->skb_iif;
966
967	fl6.flowi6_oif = oif;
968	}
969
970	if (sk) {
971	if (sk->sk_state == TCP_TIME_WAIT)
972	mark = inet_twsk(sk)->tw_mark;
973	else
974	mark = READ_ONCE(sk->sk_mark);
975	skb_set_delivery_time(skb: buff, kt: tcp_transmit_time(sk), mono: true);
976	}
977	if (txhash) {
978	/* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
979	skb_set_hash(skb: buff, hash: txhash, type: PKT_HASH_TYPE_L4);
980	}
981	fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
982	fl6.fl6_dport = t1->dest;
983	fl6.fl6_sport = t1->source;
984	fl6.flowi6_uid = sock_net_uid(net, sk: sk && sk_fullsock(sk) ? sk : NULL);
985	security_skb_classify_flow(skb, flic: flowi6_to_flowi_common(fl6: &fl6));
986
987	/* Pass a socket to ip6_dst_lookup either it is for RST
988	* Underlying function will use this to retrieve the network
989	* namespace
990	*/
991	if (sk && sk->sk_state != TCP_TIME_WAIT)
992	dst = ip6_dst_lookup_flow(net, sk, fl6: &fl6, NULL); /*sk's xfrm_policy can be referred*/
993	else
994	dst = ip6_dst_lookup_flow(net, sk: ctl_sk, fl6: &fl6, NULL);
995	if (!IS_ERR(ptr: dst)) {
996	skb_dst_set(skb: buff, dst);
997	ip6_xmit(sk: ctl_sk, skb: buff, fl6: &fl6, mark: fl6.flowi6_mark, NULL,
998	tclass: tclass & ~INET_ECN_MASK, priority);
999	TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
1000	if (rst)
1001	TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
1002	return;
1003	}
1004
1005	kfree_skb(skb: buff);
1006	}
1007
1008	static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
1009	{
1010	const struct tcphdr *th = tcp_hdr(skb);
1011	struct ipv6hdr *ipv6h = ipv6_hdr(skb);
1012	const __u8 *md5_hash_location = NULL;
1013	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1014	bool allocated_traffic_key = false;
1015	#endif
1016	const struct tcp_ao_hdr *aoh;
1017	struct tcp_key key = {};
1018	u32 seq = 0, ack_seq = 0;
1019	__be32 label = 0;
1020	u32 priority = 0;
1021	struct net *net;
1022	u32 txhash = 0;
1023	int oif = 0;
1024	#ifdef CONFIG_TCP_MD5SIG
1025	unsigned char newhash[16];
1026	int genhash;
1027	struct sock *sk1 = NULL;
1028	#endif
1029
1030	if (th->rst)
1031	return;
1032
1033	/* If sk not NULL, it means we did a successful lookup and incoming
1034	* route had to be correct. prequeue might have dropped our dst.
1035	*/
1036	if (!sk && !ipv6_unicast_destination(skb))
1037	return;
1038
1039	net = sk ? sock_net(sk) : dev_net(dev: skb_dst(skb)->dev);
1040	/* Invalid TCP option size or twice included auth */
1041	if (tcp_parse_auth_options(th, md5_hash: &md5_hash_location, aoh: &aoh))
1042	return;
1043	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1044	rcu_read_lock();
1045	#endif
1046	#ifdef CONFIG_TCP_MD5SIG
1047	if (sk && sk_fullsock(sk)) {
1048	int l3index;
1049
1050	/* sdif set, means packet ingressed via a device
1051	* in an L3 domain and inet_iif is set to it.
1052	*/
1053	l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1054	key.md5_key = tcp_v6_md5_do_lookup(sk, addr: &ipv6h->saddr, l3index);
1055	if (key.md5_key)
1056	key.type = TCP_KEY_MD5;
1057	} else if (md5_hash_location) {
1058	int dif = tcp_v6_iif_l3_slave(skb);
1059	int sdif = tcp_v6_sdif(skb);
1060	int l3index;
1061
1062	/*
1063	* active side is lost. Try to find listening socket through
1064	* source port, and then find md5 key through listening socket.
1065	* we are not loose security here:
1066	* Incoming packet is checked with md5 hash with finding key,
1067	* no RST generated if md5 hash doesn't match.
1068	*/
1069	sk1 = inet6_lookup_listener(net, hashinfo: net->ipv4.tcp_death_row.hashinfo,
1070	NULL, doff: 0, saddr: &ipv6h->saddr, sport: th->source,
1071	daddr: &ipv6h->daddr, ntohs(th->source),
1072	dif, sdif);
1073	if (!sk1)
1074	goto out;
1075
1076	/* sdif set, means packet ingressed via a device
1077	* in an L3 domain and dif is set to it.
1078	*/
1079	l3index = tcp_v6_sdif(skb) ? dif : 0;
1080
1081	key.md5_key = tcp_v6_md5_do_lookup(sk: sk1, addr: &ipv6h->saddr, l3index);
1082	if (!key.md5_key)
1083	goto out;
1084	key.type = TCP_KEY_MD5;
1085
1086	genhash = tcp_v6_md5_hash_skb(md5_hash: newhash, key: key.md5_key, NULL, skb);
1087	if (genhash || memcmp(p: md5_hash_location, q: newhash, size: 16) != 0)
1088	goto out;
1089	}
1090	#endif
1091
1092	if (th->ack)
1093	seq = ntohl(th->ack_seq);
1094	else
1095	ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1096	(th->doff << 2);
1097
1098	#ifdef CONFIG_TCP_AO
1099	if (aoh) {
1100	int l3index;
1101
1102	l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1103	if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
1104	key: &key.ao_key, traffic_key: &key.traffic_key,
1105	allocated_traffic_key: &allocated_traffic_key,
1106	keyid: &key.rcv_next, sne: &key.sne))
1107	goto out;
1108	key.type = TCP_KEY_AO;
1109	}
1110	#endif
1111
1112	if (sk) {
1113	oif = sk->sk_bound_dev_if;
1114	if (sk_fullsock(sk)) {
1115	trace_tcp_send_reset(sk, skb);
1116	if (inet6_test_bit(REPFLOW, sk))
1117	label = ip6_flowlabel(hdr: ipv6h);
1118	priority = READ_ONCE(sk->sk_priority);
1119	txhash = sk->sk_txhash;
1120	}
1121	if (sk->sk_state == TCP_TIME_WAIT) {
1122	label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1123	priority = inet_twsk(sk)->tw_priority;
1124	txhash = inet_twsk(sk)->tw_txhash;
1125	}
1126	} else {
1127	if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
1128	label = ip6_flowlabel(hdr: ipv6h);
1129	}
1130
1131	tcp_v6_send_response(sk, skb, seq, ack: ack_seq, win: 0, tsval: 0, tsecr: 0, oif, rst: 1,
1132	tclass: ipv6_get_dsfield(ipv6h), label, priority, txhash,
1133	key: &key);
1134
1135	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1136	out:
1137	if (allocated_traffic_key)
1138	kfree(objp: key.traffic_key);
1139	rcu_read_unlock();
1140	#endif
1141	}
1142
1143	static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1144	u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1145	struct tcp_key *key, u8 tclass,
1146	__be32 label, u32 priority, u32 txhash)
1147	{
1148	tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, rst: 0,
1149	tclass, label, priority, txhash, key);
1150	}
1151
1152	static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
1153	{
1154	struct inet_timewait_sock *tw = inet_twsk(sk);
1155	struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1156	struct tcp_key key = {};
1157	#ifdef CONFIG_TCP_AO
1158	struct tcp_ao_info *ao_info;
1159
1160	if (static_branch_unlikely(&tcp_ao_needed.key)) {
1161
1162	/* FIXME: the segment to-be-acked is not verified yet */
1163	ao_info = rcu_dereference(tcptw->ao_info);
1164	if (ao_info) {
1165	const struct tcp_ao_hdr *aoh;
1166
1167	/* Invalid TCP option size or twice included auth */
1168	if (tcp_parse_auth_options(th: tcp_hdr(skb), NULL, aoh: &aoh))
1169	goto out;
1170	if (aoh)
1171	key.ao_key = tcp_ao_established_key(ao: ao_info,
1172	sndid: aoh->rnext_keyid, rcvid: -1);
1173	}
1174	}
1175	if (key.ao_key) {
1176	struct tcp_ao_key *rnext_key;
1177
1178	key.traffic_key = snd_other_key(key: key.ao_key);
1179	/* rcv_next switches to our rcv_next */
1180	rnext_key = READ_ONCE(ao_info->rnext_key);
1181	key.rcv_next = rnext_key->rcvid;
1182	key.sne = READ_ONCE(ao_info->snd_sne);
1183	key.type = TCP_KEY_AO;
1184	#else
1185	if (0) {
1186	#endif
1187	#ifdef CONFIG_TCP_MD5SIG
1188	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1189	key.md5_key = tcp_twsk_md5_key(tcptw);
1190	if (key.md5_key)
1191	key.type = TCP_KEY_MD5;
1192	#endif
1193	}
1194
1195	tcp_v6_send_ack(sk, skb, seq: tcptw->tw_snd_nxt, ack: tcptw->tw_rcv_nxt,
1196	win: tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1197	tsval: tcp_tw_tsval(tcptw),
1198	tsecr: tcptw->tw_ts_recent, oif: tw->tw_bound_dev_if, key: &key,
1199	tclass: tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), priority: tw->tw_priority,
1200	txhash: tw->tw_txhash);
1201
1202	#ifdef CONFIG_TCP_AO
1203	out:
1204	#endif
1205	inet_twsk_put(tw);
1206	}
1207
1208	static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1209	struct request_sock *req)
1210	{
1211	struct tcp_key key = {};
1212
1213	#ifdef CONFIG_TCP_AO
1214	if (static_branch_unlikely(&tcp_ao_needed.key) &&
1215	tcp_rsk_used_ao(req)) {
1216	const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
1217	const struct tcp_ao_hdr *aoh;
1218	int l3index;
1219
1220	l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1221	/* Invalid TCP option size or twice included auth */
1222	if (tcp_parse_auth_options(th: tcp_hdr(skb), NULL, aoh: &aoh))
1223	return;
1224	if (!aoh)
1225	return;
1226	key.ao_key = tcp_ao_do_lookup(sk, l3index,
1227	addr: (union tcp_ao_addr *)addr,
1228	AF_INET6, sndid: aoh->rnext_keyid, rcvid: -1);
1229	if (unlikely(!key.ao_key)) {
1230	/* Send ACK with any matching MKT for the peer */
1231	key.ao_key = tcp_ao_do_lookup(sk, l3index,
1232	addr: (union tcp_ao_addr *)addr,
1233	AF_INET6, sndid: -1, rcvid: -1);
1234	/* Matching key disappeared (user removed the key?)
1235	* let the handshake timeout.
1236	*/
1237	if (!key.ao_key) {
1238	net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
1239	addr,
1240	ntohs(tcp_hdr(skb)->source),
1241	&ipv6_hdr(skb)->daddr,
1242	ntohs(tcp_hdr(skb)->dest));
1243	return;
1244	}
1245	}
1246	key.traffic_key = kmalloc(size: tcp_ao_digest_size(key: key.ao_key), GFP_ATOMIC);
1247	if (!key.traffic_key)
1248	return;
1249
1250	key.type = TCP_KEY_AO;
1251	key.rcv_next = aoh->keyid;
1252	tcp_v6_ao_calc_key_rsk(mkt: key.ao_key, key: key.traffic_key, req);
1253	#else
1254	if (0) {
1255	#endif
1256	#ifdef CONFIG_TCP_MD5SIG
1257	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1258	int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1259
1260	key.md5_key = tcp_v6_md5_do_lookup(sk, addr: &ipv6_hdr(skb)->saddr,
1261	l3index);
1262	if (key.md5_key)
1263	key.type = TCP_KEY_MD5;
1264	#endif
1265	}
1266
1267	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1268	* sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1269	*/
1270	/* RFC 7323 2.3
1271	* The window field (SEG.WND) of every outgoing segment, with the
1272	* exception of <SYN> segments, MUST be right-shifted by
1273	* Rcv.Wind.Shift bits:
1274	*/
1275	tcp_v6_send_ack(sk, skb, seq: (sk->sk_state == TCP_LISTEN) ?
1276	tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1277	ack: tcp_rsk(req)->rcv_nxt,
1278	win: req->rsk_rcv_wnd >> inet_rsk(sk: req)->rcv_wscale,
1279	tsval: tcp_rsk_tsval(treq: tcp_rsk(req)),
1280	READ_ONCE(req->ts_recent), oif: sk->sk_bound_dev_if,
1281	key: &key, tclass: ipv6_get_dsfield(ipv6h: ipv6_hdr(skb)), label: 0,
1282	READ_ONCE(sk->sk_priority),
1283	READ_ONCE(tcp_rsk(req)->txhash));
1284	if (tcp_key_is_ao(key: &key))
1285	kfree(objp: key.traffic_key);
1286	}
1287
1288
1289	static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1290	{
1291	#ifdef CONFIG_SYN_COOKIES
1292	const struct tcphdr *th = tcp_hdr(skb);
1293
1294	if (!th->syn)
1295	sk = cookie_v6_check(sk, skb);
1296	#endif
1297	return sk;
1298	}
1299
1300	u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1301	struct tcphdr *th, u32 *cookie)
1302	{
1303	u16 mss = 0;
1304	#ifdef CONFIG_SYN_COOKIES
1305	mss = tcp_get_syncookie_mss(rsk_ops: &tcp6_request_sock_ops,
1306	af_ops: &tcp_request_sock_ipv6_ops, sk, th);
1307	if (mss) {
1308	*cookie = __cookie_v6_init_sequence(iph, th, mssp: &mss);
1309	tcp_synq_overflow(sk);
1310	}
1311	#endif
1312	return mss;
1313	}
1314
1315	static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1316	{
1317	if (skb->protocol == htons(ETH_P_IP))
1318	return tcp_v4_conn_request(sk, skb);
1319
1320	if (!ipv6_unicast_destination(skb))
1321	goto drop;
1322
1323	if (ipv6_addr_v4mapped(a: &ipv6_hdr(skb)->saddr)) {
1324	__IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1325	return 0;
1326	}
1327
1328	return tcp_conn_request(rsk_ops: &tcp6_request_sock_ops,
1329	af_ops: &tcp_request_sock_ipv6_ops, sk, skb);
1330
1331	drop:
1332	tcp_listendrop(sk);
1333	return 0; /* don't send reset */
1334	}
1335
1336	static void tcp_v6_restore_cb(struct sk_buff *skb)
1337	{
1338	/* We need to move header back to the beginning if xfrm6_policy_check()
1339	* and tcp_v6_fill_cb() are going to be called again.
1340	* ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1341	*/
1342	memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1343	sizeof(struct inet6_skb_parm));
1344	}
1345
1346	static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1347	struct request_sock *req,
1348	struct dst_entry *dst,
1349	struct request_sock *req_unhash,
1350	bool *own_req)
1351	{
1352	struct inet_request_sock *ireq;
1353	struct ipv6_pinfo *newnp;
1354	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1355	struct ipv6_txoptions *opt;
1356	struct inet_sock *newinet;
1357	bool found_dup_sk = false;
1358	struct tcp_sock *newtp;
1359	struct sock *newsk;
1360	#ifdef CONFIG_TCP_MD5SIG
1361	struct tcp_md5sig_key *key;
1362	int l3index;
1363	#endif
1364	struct flowi6 fl6;
1365
1366	if (skb->protocol == htons(ETH_P_IP)) {
1367	/*
1368	* v6 mapped
1369	*/
1370
1371	newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst,
1372	req_unhash, own_req);
1373
1374	if (!newsk)
1375	return NULL;
1376
1377	inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1378
1379	newnp = tcp_inet6_sk(newsk);
1380	newtp = tcp_sk(newsk);
1381
1382	memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1383
1384	newnp->saddr = newsk->sk_v6_rcv_saddr;
1385
1386	inet_csk(sk: newsk)->icsk_af_ops = &ipv6_mapped;
1387	if (sk_is_mptcp(sk: newsk))
1388	mptcpv6_handle_mapped(sk: newsk, mapped: true);
1389	newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1390	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1391	newtp->af_specific = &tcp_sock_ipv6_mapped_specific;
1392	#endif
1393
1394	newnp->ipv6_mc_list = NULL;
1395	newnp->ipv6_ac_list = NULL;
1396	newnp->ipv6_fl_list = NULL;
1397	newnp->pktoptions = NULL;
1398	newnp->opt = NULL;
1399	newnp->mcast_oif = inet_iif(skb);
1400	newnp->mcast_hops = ip_hdr(skb)->ttl;
1401	newnp->rcv_flowinfo = 0;
1402	if (inet6_test_bit(REPFLOW, sk))
1403	newnp->flow_label = 0;
1404
1405	/*
1406	* No need to charge this sock to the relevant IPv6 refcnt debug socks count
1407	* here, tcp_create_openreq_child now does this for us, see the comment in
1408	* that function for the gory details. -acme
1409	*/
1410
1411	/* It is tricky place. Until this moment IPv4 tcp
1412	worked with IPv6 icsk.icsk_af_ops.
1413	Sync it now.
1414	*/
1415	tcp_sync_mss(sk: newsk, pmtu: inet_csk(sk: newsk)->icsk_pmtu_cookie);
1416
1417	return newsk;
1418	}
1419
1420	ireq = inet_rsk(sk: req);
1421
1422	if (sk_acceptq_is_full(sk))
1423	goto out_overflow;
1424
1425	if (!dst) {
1426	dst = inet6_csk_route_req(sk, fl6: &fl6, req, IPPROTO_TCP);
1427	if (!dst)
1428	goto out;
1429	}
1430
1431	newsk = tcp_create_openreq_child(sk, req, skb);
1432	if (!newsk)
1433	goto out_nonewsk;
1434
1435	/*
1436	* No need to charge this sock to the relevant IPv6 refcnt debug socks
1437	* count here, tcp_create_openreq_child now does this for us, see the
1438	* comment in that function for the gory details. -acme
1439	*/
1440
1441	newsk->sk_gso_type = SKB_GSO_TCPV6;
1442	ip6_dst_store(sk: newsk, dst, NULL, NULL);
1443	inet6_sk_rx_dst_set(sk: newsk, skb);
1444
1445	inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
1446
1447	newtp = tcp_sk(newsk);
1448	newinet = inet_sk(newsk);
1449	newnp = tcp_inet6_sk(newsk);
1450
1451	memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1452
1453	newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr;
1454	newnp->saddr = ireq->ir_v6_loc_addr;
1455	newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr;
1456	newsk->sk_bound_dev_if = ireq->ir_iif;
1457
1458	/* Now IPv6 options...
1459
1460	First: no IPv4 options.
1461	*/
1462	newinet->inet_opt = NULL;
1463	newnp->ipv6_mc_list = NULL;
1464	newnp->ipv6_ac_list = NULL;
1465	newnp->ipv6_fl_list = NULL;
1466
1467	/* Clone RX bits */
1468	newnp->rxopt.all = np->rxopt.all;
1469
1470	newnp->pktoptions = NULL;
1471	newnp->opt = NULL;
1472	newnp->mcast_oif = tcp_v6_iif(skb);
1473	newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1474	newnp->rcv_flowinfo = ip6_flowinfo(hdr: ipv6_hdr(skb));
1475	if (inet6_test_bit(REPFLOW, sk))
1476	newnp->flow_label = ip6_flowlabel(hdr: ipv6_hdr(skb));
1477
1478	/* Set ToS of the new socket based upon the value of incoming SYN.
1479	* ECT bits are set later in tcp_init_transfer().
1480	*/
1481	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1482	newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1483
1484	/* Clone native IPv6 options from listening socket (if any)
1485
1486	Yes, keeping reference count would be much more clever,
1487	but we make one more one thing there: reattach optmem
1488	to newsk.
1489	*/
1490	opt = ireq->ipv6_opt;
1491	if (!opt)
1492	opt = rcu_dereference(np->opt);
1493	if (opt) {
1494	opt = ipv6_dup_options(sk: newsk, opt);
1495	RCU_INIT_POINTER(newnp->opt, opt);
1496	}
1497	inet_csk(sk: newsk)->icsk_ext_hdr_len = 0;
1498	if (opt)
1499	inet_csk(sk: newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1500	opt->opt_flen;
1501
1502	tcp_ca_openreq_child(sk: newsk, dst);
1503
1504	tcp_sync_mss(sk: newsk, pmtu: dst_mtu(dst));
1505	newtp->advmss = tcp_mss_clamp(tcp_sk(sk), mss: dst_metric_advmss(dst));
1506
1507	tcp_initialize_rcv_mss(sk: newsk);
1508
1509	newinet->inet_daddr = newinet->inet_saddr = LOOPBACK4_IPV6;
1510	newinet->inet_rcv_saddr = LOOPBACK4_IPV6;
1511
1512	#ifdef CONFIG_TCP_MD5SIG
1513	l3index = l3mdev_master_ifindex_by_index(net: sock_net(sk), ifindex: ireq->ir_iif);
1514
1515	if (!tcp_rsk_used_ao(req)) {
1516	/* Copy over the MD5 key from the original socket */
1517	key = tcp_v6_md5_do_lookup(sk, addr: &newsk->sk_v6_daddr, l3index);
1518	if (key) {
1519	const union tcp_md5_addr *addr;
1520
1521	addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
1522	if (tcp_md5_key_copy(sk: newsk, addr, AF_INET6, prefixlen: 128, l3index, key)) {
1523	inet_csk_prepare_forced_close(sk: newsk);
1524	tcp_done(sk: newsk);
1525	goto out;
1526	}
1527	}
1528	}
1529	#endif
1530	#ifdef CONFIG_TCP_AO
1531	/* Copy over tcp_ao_info if any */
1532	if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
1533	goto out; /* OOM */
1534	#endif
1535
1536	if (__inet_inherit_port(sk, child: newsk) < 0) {
1537	inet_csk_prepare_forced_close(sk: newsk);
1538	tcp_done(sk: newsk);
1539	goto out;
1540	}
1541	*own_req = inet_ehash_nolisten(sk: newsk, osk: req_to_sk(req: req_unhash),
1542	found_dup_sk: &found_dup_sk);
1543	if (*own_req) {
1544	tcp_move_syn(tp: newtp, req);
1545
1546	/* Clone pktoptions received with SYN, if we own the req */
1547	if (ireq->pktopts) {
1548	newnp->pktoptions = skb_clone_and_charge_r(skb: ireq->pktopts, sk: newsk);
1549	consume_skb(skb: ireq->pktopts);
1550	ireq->pktopts = NULL;
1551	if (newnp->pktoptions)
1552	tcp_v6_restore_cb(skb: newnp->pktoptions);
1553	}
1554	} else {
1555	if (!req_unhash && found_dup_sk) {
1556	/* This code path should only be executed in the
1557	* syncookie case only
1558	*/
1559	bh_unlock_sock(newsk);
1560	sock_put(sk: newsk);
1561	newsk = NULL;
1562	}
1563	}
1564
1565	return newsk;
1566
1567	out_overflow:
1568	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1569	out_nonewsk:
1570	dst_release(dst);
1571	out:
1572	tcp_listendrop(sk);
1573	return NULL;
1574	}
1575
1576	INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1577	u32));
1578	/* The socket must have it's spinlock held when we get
1579	* here, unless it is a TCP_LISTEN socket.
1580	*
1581	* We have a potential double-lock case here, so even when
1582	* doing backlog processing we use the BH locking scheme.
1583	* This is because we cannot sleep with the original spinlock
1584	* held.
1585	*/
1586	INDIRECT_CALLABLE_SCOPE
1587	int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1588	{
1589	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1590	struct sk_buff *opt_skb = NULL;
1591	enum skb_drop_reason reason;
1592	struct tcp_sock *tp;
1593
1594	/* Imagine: socket is IPv6. IPv4 packet arrives,
1595	goes to IPv4 receive handler and backlogged.
1596	From backlog it always goes here. Kerboom...
1597	Fortunately, tcp_rcv_established and rcv_established
1598	handle them correctly, but it is not case with
1599	tcp_v6_hnd_req and tcp_v6_send_reset(). --ANK
1600	*/
1601
1602	if (skb->protocol == htons(ETH_P_IP))
1603	return tcp_v4_do_rcv(sk, skb);
1604
1605	/*
1606	* socket locking is here for SMP purposes as backlog rcv
1607	* is currently called with bh processing disabled.
1608	*/
1609
1610	/* Do Stevens' IPV6_PKTOPTIONS.
1611
1612	Yes, guys, it is the only place in our code, where we
1613	may make it not affecting IPv4.
1614	The rest of code is protocol independent,
1615	and I do not like idea to uglify IPv4.
1616
1617	Actually, all the idea behind IPV6_PKTOPTIONS
1618	looks not very well thought. For now we latch
1619	options, received in the last packet, enqueued
1620	by tcp. Feel free to propose better solution.
1621	--ANK (980728)
1622	*/
1623	if (np->rxopt.all)
1624	opt_skb = skb_clone_and_charge_r(skb, sk);
1625
1626	reason = SKB_DROP_REASON_NOT_SPECIFIED;
1627	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1628	struct dst_entry *dst;
1629
1630	dst = rcu_dereference_protected(sk->sk_rx_dst,
1631	lockdep_sock_is_held(sk));
1632
1633	sock_rps_save_rxhash(sk, skb);
1634	sk_mark_napi_id(sk, skb);
1635	if (dst) {
1636	if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1637	INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1638	dst, sk->sk_rx_dst_cookie) == NULL) {
1639	RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1640	dst_release(dst);
1641	}
1642	}
1643
1644	tcp_rcv_established(sk, skb);
1645	if (opt_skb)
1646	goto ipv6_pktoptions;
1647	return 0;
1648	}
1649
1650	if (tcp_checksum_complete(skb))
1651	goto csum_err;
1652
1653	if (sk->sk_state == TCP_LISTEN) {
1654	struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1655
1656	if (!nsk)
1657	goto discard;
1658
1659	if (nsk != sk) {
1660	if (tcp_child_process(parent: sk, child: nsk, skb))
1661	goto reset;
1662	if (opt_skb)
1663	__kfree_skb(skb: opt_skb);
1664	return 0;
1665	}
1666	} else
1667	sock_rps_save_rxhash(sk, skb);
1668
1669	if (tcp_rcv_state_process(sk, skb))
1670	goto reset;
1671	if (opt_skb)
1672	goto ipv6_pktoptions;
1673	return 0;
1674
1675	reset:
1676	tcp_v6_send_reset(sk, skb);
1677	discard:
1678	if (opt_skb)
1679	__kfree_skb(skb: opt_skb);
1680	kfree_skb_reason(skb, reason);
1681	return 0;
1682	csum_err:
1683	reason = SKB_DROP_REASON_TCP_CSUM;
1684	trace_tcp_bad_csum(skb);
1685	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1686	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1687	goto discard;
1688
1689
1690	ipv6_pktoptions:
1691	/* Do you ask, what is it?
1692
1693	1. skb was enqueued by tcp.
1694	2. skb is added to tail of read queue, rather than out of order.
1695	3. socket is not in passive state.
1696	4. Finally, it really contains options, which user wants to receive.
1697	*/
1698	tp = tcp_sk(sk);
1699	if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1700	!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1701	if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1702	np->mcast_oif = tcp_v6_iif(skb: opt_skb);
1703	if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1704	WRITE_ONCE(np->mcast_hops,
1705	ipv6_hdr(opt_skb)->hop_limit);
1706	if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1707	np->rcv_flowinfo = ip6_flowinfo(hdr: ipv6_hdr(skb: opt_skb));
1708	if (inet6_test_bit(REPFLOW, sk))
1709	np->flow_label = ip6_flowlabel(hdr: ipv6_hdr(skb: opt_skb));
1710	if (ipv6_opt_accepted(sk, skb: opt_skb, opt: &TCP_SKB_CB(opt_skb)->header.h6)) {
1711	tcp_v6_restore_cb(skb: opt_skb);
1712	opt_skb = xchg(&np->pktoptions, opt_skb);
1713	} else {
1714	__kfree_skb(skb: opt_skb);
1715	opt_skb = xchg(&np->pktoptions, NULL);
1716	}
1717	}
1718
1719	consume_skb(skb: opt_skb);
1720	return 0;
1721	}
1722
1723	static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1724	const struct tcphdr *th)
1725	{
1726	/* This is tricky: we move IP6CB at its correct location into
1727	* TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1728	* _decode_session6() uses IP6CB().
1729	* barrier() makes sure compiler won't play aliasing games.
1730	*/
1731	memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1732	sizeof(struct inet6_skb_parm));
1733	barrier();
1734
1735	TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1736	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1737	skb->len - th->doff*4);
1738	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1739	TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1740	TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1741	TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(ipv6h: hdr);
1742	TCP_SKB_CB(skb)->sacked = 0;
1743	TCP_SKB_CB(skb)->has_rxtstamp =
1744	skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1745	}
1746
1747	INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1748	{
1749	enum skb_drop_reason drop_reason;
1750	int sdif = inet6_sdif(skb);
1751	int dif = inet6_iif(skb);
1752	const struct tcphdr *th;
1753	const struct ipv6hdr *hdr;
1754	bool refcounted;
1755	struct sock *sk;
1756	int ret;
1757	struct net *net = dev_net(dev: skb->dev);
1758
1759	drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1760	if (skb->pkt_type != PACKET_HOST)
1761	goto discard_it;
1762
1763	/*
1764	* Count it even if it's bad.
1765	*/
1766	__TCP_INC_STATS(net, TCP_MIB_INSEGS);
1767
1768	if (!pskb_may_pull(skb, len: sizeof(struct tcphdr)))
1769	goto discard_it;
1770
1771	th = (const struct tcphdr *)skb->data;
1772
1773	if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1774	drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1775	goto bad_packet;
1776	}
1777	if (!pskb_may_pull(skb, len: th->doff*4))
1778	goto discard_it;
1779
1780	if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1781	goto csum_error;
1782
1783	th = (const struct tcphdr *)skb->data;
1784	hdr = ipv6_hdr(skb);
1785
1786	lookup:
1787	sk = __inet6_lookup_skb(hashinfo: net->ipv4.tcp_death_row.hashinfo, skb, doff: __tcp_hdrlen(th),
1788	sport: th->source, dport: th->dest, iif: inet6_iif(skb), sdif,
1789	refcounted: &refcounted);
1790	if (!sk)
1791	goto no_tcp_socket;
1792
1793	process:
1794	if (sk->sk_state == TCP_TIME_WAIT)
1795	goto do_time_wait;
1796
1797	if (sk->sk_state == TCP_NEW_SYN_RECV) {
1798	struct request_sock *req = inet_reqsk(sk);
1799	bool req_stolen = false;
1800	struct sock *nsk;
1801
1802	sk = req->rsk_listener;
1803	if (!xfrm6_policy_check(sk, dir: XFRM_POLICY_IN, skb))
1804	drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1805	else
1806	drop_reason = tcp_inbound_hash(sk, req, skb,
1807	saddr: &hdr->saddr, daddr: &hdr->daddr,
1808	AF_INET6, dif, sdif);
1809	if (drop_reason) {
1810	sk_drops_add(sk, skb);
1811	reqsk_put(req);
1812	goto discard_it;
1813	}
1814	if (tcp_checksum_complete(skb)) {
1815	reqsk_put(req);
1816	goto csum_error;
1817	}
1818	if (unlikely(sk->sk_state != TCP_LISTEN)) {
1819	nsk = reuseport_migrate_sock(sk, migrating_sk: req_to_sk(req), skb);
1820	if (!nsk) {
1821	inet_csk_reqsk_queue_drop_and_put(sk, req);
1822	goto lookup;
1823	}
1824	sk = nsk;
1825	/* reuseport_migrate_sock() has already held one sk_refcnt
1826	* before returning.
1827	*/
1828	} else {
1829	sock_hold(sk);
1830	}
1831	refcounted = true;
1832	nsk = NULL;
1833	if (!tcp_filter(sk, skb)) {
1834	th = (const struct tcphdr *)skb->data;
1835	hdr = ipv6_hdr(skb);
1836	tcp_v6_fill_cb(skb, hdr, th);
1837	nsk = tcp_check_req(sk, skb, req, fastopen: false, lost_race: &req_stolen);
1838	} else {
1839	drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1840	}
1841	if (!nsk) {
1842	reqsk_put(req);
1843	if (req_stolen) {
1844	/* Another cpu got exclusive access to req
1845	* and created a full blown socket.
1846	* Try to feed this packet to this socket
1847	* instead of discarding it.
1848	*/
1849	tcp_v6_restore_cb(skb);
1850	sock_put(sk);
1851	goto lookup;
1852	}
1853	goto discard_and_relse;
1854	}
1855	nf_reset_ct(skb);
1856	if (nsk == sk) {
1857	reqsk_put(req);
1858	tcp_v6_restore_cb(skb);
1859	} else if (tcp_child_process(parent: sk, child: nsk, skb)) {
1860	tcp_v6_send_reset(sk: nsk, skb);
1861	goto discard_and_relse;
1862	} else {
1863	sock_put(sk);
1864	return 0;
1865	}
1866	}
1867
1868	if (static_branch_unlikely(&ip6_min_hopcount)) {
1869	/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
1870	if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
1871	__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1872	drop_reason = SKB_DROP_REASON_TCP_MINTTL;
1873	goto discard_and_relse;
1874	}
1875	}
1876
1877	if (!xfrm6_policy_check(sk, dir: XFRM_POLICY_IN, skb)) {
1878	drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1879	goto discard_and_relse;
1880	}
1881
1882	drop_reason = tcp_inbound_hash(sk, NULL, skb, saddr: &hdr->saddr, daddr: &hdr->daddr,
1883	AF_INET6, dif, sdif);
1884	if (drop_reason)
1885	goto discard_and_relse;
1886
1887	nf_reset_ct(skb);
1888
1889	if (tcp_filter(sk, skb)) {
1890	drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1891	goto discard_and_relse;
1892	}
1893	th = (const struct tcphdr *)skb->data;
1894	hdr = ipv6_hdr(skb);
1895	tcp_v6_fill_cb(skb, hdr, th);
1896
1897	skb->dev = NULL;
1898
1899	if (sk->sk_state == TCP_LISTEN) {
1900	ret = tcp_v6_do_rcv(sk, skb);
1901	goto put_and_return;
1902	}
1903
1904	sk_incoming_cpu_update(sk);
1905
1906	bh_lock_sock_nested(sk);
1907	tcp_segs_in(tcp_sk(sk), skb);
1908	ret = 0;
1909	if (!sock_owned_by_user(sk)) {
1910	ret = tcp_v6_do_rcv(sk, skb);
1911	} else {
1912	if (tcp_add_backlog(sk, skb, reason: &drop_reason))
1913	goto discard_and_relse;
1914	}
1915	bh_unlock_sock(sk);
1916	put_and_return:
1917	if (refcounted)
1918	sock_put(sk);
1919	return ret ? -1 : 0;
1920
1921	no_tcp_socket:
1922	drop_reason = SKB_DROP_REASON_NO_SOCKET;
1923	if (!xfrm6_policy_check(NULL, dir: XFRM_POLICY_IN, skb))
1924	goto discard_it;
1925
1926	tcp_v6_fill_cb(skb, hdr, th);
1927
1928	if (tcp_checksum_complete(skb)) {
1929	csum_error:
1930	drop_reason = SKB_DROP_REASON_TCP_CSUM;
1931	trace_tcp_bad_csum(skb);
1932	__TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1933	bad_packet:
1934	__TCP_INC_STATS(net, TCP_MIB_INERRS);
1935	} else {
1936	tcp_v6_send_reset(NULL, skb);
1937	}
1938
1939	discard_it:
1940	SKB_DR_OR(drop_reason, NOT_SPECIFIED);
1941	kfree_skb_reason(skb, reason: drop_reason);
1942	return 0;
1943
1944	discard_and_relse:
1945	sk_drops_add(sk, skb);
1946	if (refcounted)
1947	sock_put(sk);
1948	goto discard_it;
1949
1950	do_time_wait:
1951	if (!xfrm6_policy_check(NULL, dir: XFRM_POLICY_IN, skb)) {
1952	drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1953	inet_twsk_put(tw: inet_twsk(sk));
1954	goto discard_it;
1955	}
1956
1957	tcp_v6_fill_cb(skb, hdr, th);
1958
1959	if (tcp_checksum_complete(skb)) {
1960	inet_twsk_put(tw: inet_twsk(sk));
1961	goto csum_error;
1962	}
1963
1964	switch (tcp_timewait_state_process(tw: inet_twsk(sk), skb, th)) {
1965	case TCP_TW_SYN:
1966	{
1967	struct sock *sk2;
1968
1969	sk2 = inet6_lookup_listener(net, hashinfo: net->ipv4.tcp_death_row.hashinfo,
1970	skb, doff: __tcp_hdrlen(th),
1971	saddr: &ipv6_hdr(skb)->saddr, sport: th->source,
1972	daddr: &ipv6_hdr(skb)->daddr,
1973	ntohs(th->dest),
1974	dif: tcp_v6_iif_l3_slave(skb),
1975	sdif);
1976	if (sk2) {
1977	struct inet_timewait_sock *tw = inet_twsk(sk);
1978	inet_twsk_deschedule_put(tw);
1979	sk = sk2;
1980	tcp_v6_restore_cb(skb);
1981	refcounted = false;
1982	goto process;
1983	}
1984	}
1985	/* to ACK */
1986	fallthrough;
1987	case TCP_TW_ACK:
1988	tcp_v6_timewait_ack(sk, skb);
1989	break;
1990	case TCP_TW_RST:
1991	tcp_v6_send_reset(sk, skb);
1992	inet_twsk_deschedule_put(tw: inet_twsk(sk));
1993	goto discard_it;
1994	case TCP_TW_SUCCESS:
1995	;
1996	}
1997	goto discard_it;
1998	}
1999
2000	void tcp_v6_early_demux(struct sk_buff *skb)
2001	{
2002	struct net *net = dev_net(dev: skb->dev);
2003	const struct ipv6hdr *hdr;
2004	const struct tcphdr *th;
2005	struct sock *sk;
2006
2007	if (skb->pkt_type != PACKET_HOST)
2008	return;
2009
2010	if (!pskb_may_pull(skb, len: skb_transport_offset(skb) + sizeof(struct tcphdr)))
2011	return;
2012
2013	hdr = ipv6_hdr(skb);
2014	th = tcp_hdr(skb);
2015
2016	if (th->doff < sizeof(struct tcphdr) / 4)
2017	return;
2018
2019	/* Note : We use inet6_iif() here, not tcp_v6_iif() */
2020	sk = __inet6_lookup_established(net, hashinfo: net->ipv4.tcp_death_row.hashinfo,
2021	saddr: &hdr->saddr, sport: th->source,
2022	daddr: &hdr->daddr, ntohs(th->dest),
2023	dif: inet6_iif(skb), sdif: inet6_sdif(skb));
2024	if (sk) {
2025	skb->sk = sk;
2026	skb->destructor = sock_edemux;
2027	if (sk_fullsock(sk)) {
2028	struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
2029
2030	if (dst)
2031	dst = dst_check(dst, cookie: sk->sk_rx_dst_cookie);
2032	if (dst &&
2033	sk->sk_rx_dst_ifindex == skb->skb_iif)
2034	skb_dst_set_noref(skb, dst);
2035	}
2036	}
2037	}
2038
2039	static struct timewait_sock_ops tcp6_timewait_sock_ops = {
2040	.twsk_obj_size = sizeof(struct tcp6_timewait_sock),
2041	.twsk_unique = tcp_twsk_unique,
2042	.twsk_destructor = tcp_twsk_destructor,
2043	};
2044
2045	INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
2046	{
2047	__tcp_v6_send_check(skb, saddr: &sk->sk_v6_rcv_saddr, daddr: &sk->sk_v6_daddr);
2048	}
2049
2050	const struct inet_connection_sock_af_ops ipv6_specific = {
2051	.queue_xmit = inet6_csk_xmit,
2052	.send_check = tcp_v6_send_check,
2053	.rebuild_header = inet6_sk_rebuild_header,
2054	.sk_rx_dst_set = inet6_sk_rx_dst_set,
2055	.conn_request = tcp_v6_conn_request,
2056	.syn_recv_sock = tcp_v6_syn_recv_sock,
2057	.net_header_len = sizeof(struct ipv6hdr),
2058	.setsockopt = ipv6_setsockopt,
2059	.getsockopt = ipv6_getsockopt,
2060	.addr2sockaddr = inet6_csk_addr2sockaddr,
2061	.sockaddr_len = sizeof(struct sockaddr_in6),
2062	.mtu_reduced = tcp_v6_mtu_reduced,
2063	};
2064
2065	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2066	static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
2067	#ifdef CONFIG_TCP_MD5SIG
2068	.md5_lookup = tcp_v6_md5_lookup,
2069	.calc_md5_hash = tcp_v6_md5_hash_skb,
2070	.md5_parse = tcp_v6_parse_md5_keys,
2071	#endif
2072	#ifdef CONFIG_TCP_AO
2073	.ao_lookup = tcp_v6_ao_lookup,
2074	.calc_ao_hash = tcp_v6_ao_hash_skb,
2075	.ao_parse = tcp_v6_parse_ao,
2076	.ao_calc_key_sk = tcp_v6_ao_calc_key_sk,
2077	#endif
2078	};
2079	#endif
2080
2081	/*
2082	* TCP over IPv4 via INET6 API
2083	*/
2084	static const struct inet_connection_sock_af_ops ipv6_mapped = {
2085	.queue_xmit = ip_queue_xmit,
2086	.send_check = tcp_v4_send_check,
2087	.rebuild_header = inet_sk_rebuild_header,
2088	.sk_rx_dst_set = inet_sk_rx_dst_set,
2089	.conn_request = tcp_v6_conn_request,
2090	.syn_recv_sock = tcp_v6_syn_recv_sock,
2091	.net_header_len = sizeof(struct iphdr),
2092	.setsockopt = ipv6_setsockopt,
2093	.getsockopt = ipv6_getsockopt,
2094	.addr2sockaddr = inet6_csk_addr2sockaddr,
2095	.sockaddr_len = sizeof(struct sockaddr_in6),
2096	.mtu_reduced = tcp_v4_mtu_reduced,
2097	};
2098
2099	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2100	static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
2101	#ifdef CONFIG_TCP_MD5SIG
2102	.md5_lookup = tcp_v4_md5_lookup,
2103	.calc_md5_hash = tcp_v4_md5_hash_skb,
2104	.md5_parse = tcp_v6_parse_md5_keys,
2105	#endif
2106	#ifdef CONFIG_TCP_AO
2107	.ao_lookup = tcp_v6_ao_lookup,
2108	.calc_ao_hash = tcp_v4_ao_hash_skb,
2109	.ao_parse = tcp_v6_parse_ao,
2110	.ao_calc_key_sk = tcp_v4_ao_calc_key_sk,
2111	#endif
2112	};
2113	#endif
2114
2115	/* NOTE: A lot of things set to zero explicitly by call to
2116	* sk_alloc() so need not be done here.
2117	*/
2118	static int tcp_v6_init_sock(struct sock *sk)
2119	{
2120	struct inet_connection_sock *icsk = inet_csk(sk);
2121
2122	tcp_init_sock(sk);
2123
2124	icsk->icsk_af_ops = &ipv6_specific;
2125
2126	#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2127	tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
2128	#endif
2129
2130	return 0;
2131	}
2132
2133	#ifdef CONFIG_PROC_FS
2134	/* Proc filesystem TCPv6 sock list dumping. */
2135	static void get_openreq6(struct seq_file *seq,
2136	const struct request_sock *req, int i)
2137	{
2138	long ttd = req->rsk_timer.expires - jiffies;
2139	const struct in6_addr *src = &inet_rsk(sk: req)->ir_v6_loc_addr;
2140	const struct in6_addr *dest = &inet_rsk(sk: req)->ir_v6_rmt_addr;
2141
2142	if (ttd < 0)
2143	ttd = 0;
2144
2145	seq_printf(m: seq,
2146	fmt: "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2147	"%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
2148	i,
2149	src->s6_addr32[0], src->s6_addr32[1],
2150	src->s6_addr32[2], src->s6_addr32[3],
2151	inet_rsk(sk: req)->ir_num,
2152	dest->s6_addr32[0], dest->s6_addr32[1],
2153	dest->s6_addr32[2], dest->s6_addr32[3],
2154	ntohs(inet_rsk(req)->ir_rmt_port),
2155	TCP_SYN_RECV,
2156	0, 0, /* could print option size, but that is af dependent. */
2157	1, /* timers active (only the expire timer) */
2158	jiffies_to_clock_t(x: ttd),
2159	req->num_timeout,
2160	from_kuid_munged(to: seq_user_ns(seq),
2161	uid: sock_i_uid(sk: req->rsk_listener)),
2162	0, /* non standard timer */
2163	0, /* open_requests have no inode */
2164	0, req);
2165	}
2166
2167	static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
2168	{
2169	const struct in6_addr *dest, *src;
2170	__u16 destp, srcp;
2171	int timer_active;
2172	unsigned long timer_expires;
2173	const struct inet_sock *inet = inet_sk(sp);
2174	const struct tcp_sock *tp = tcp_sk(sp);
2175	const struct inet_connection_sock *icsk = inet_csk(sk: sp);
2176	const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2177	int rx_queue;
2178	int state;
2179
2180	dest = &sp->sk_v6_daddr;
2181	src = &sp->sk_v6_rcv_saddr;
2182	destp = ntohs(inet->inet_dport);
2183	srcp = ntohs(inet->inet_sport);
2184
2185	if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2186	icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2187	icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2188	timer_active = 1;
2189	timer_expires = icsk->icsk_timeout;
2190	} else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2191	timer_active = 4;
2192	timer_expires = icsk->icsk_timeout;
2193	} else if (timer_pending(timer: &sp->sk_timer)) {
2194	timer_active = 2;
2195	timer_expires = sp->sk_timer.expires;
2196	} else {
2197	timer_active = 0;
2198	timer_expires = jiffies;
2199	}
2200
2201	state = inet_sk_state_load(sk: sp);
2202	if (state == TCP_LISTEN)
2203	rx_queue = READ_ONCE(sp->sk_ack_backlog);
2204	else
2205	/* Because we don't lock the socket,
2206	* we might find a transient negative value.
2207	*/
2208	rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2209	READ_ONCE(tp->copied_seq), 0);
2210
2211	seq_printf(m: seq,
2212	fmt: "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2213	"%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
2214	i,
2215	src->s6_addr32[0], src->s6_addr32[1],
2216	src->s6_addr32[2], src->s6_addr32[3], srcp,
2217	dest->s6_addr32[0], dest->s6_addr32[1],
2218	dest->s6_addr32[2], dest->s6_addr32[3], destp,
2219	state,
2220	READ_ONCE(tp->write_seq) - tp->snd_una,
2221	rx_queue,
2222	timer_active,
2223	jiffies_delta_to_clock_t(delta: timer_expires - jiffies),
2224	icsk->icsk_retransmits,
2225	from_kuid_munged(to: seq_user_ns(seq), uid: sock_i_uid(sk: sp)),
2226	icsk->icsk_probes_out,
2227	sock_i_ino(sk: sp),
2228	refcount_read(r: &sp->sk_refcnt), sp,
2229	jiffies_to_clock_t(x: icsk->icsk_rto),
2230	jiffies_to_clock_t(x: icsk->icsk_ack.ato),
2231	(icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk: sp),
2232	tcp_snd_cwnd(tp),
2233	state == TCP_LISTEN ?
2234	fastopenq->max_qlen :
2235	(tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2236	);
2237	}
2238
2239	static void get_timewait6_sock(struct seq_file *seq,
2240	struct inet_timewait_sock *tw, int i)
2241	{
2242	long delta = tw->tw_timer.expires - jiffies;
2243	const struct in6_addr *dest, *src;
2244	__u16 destp, srcp;
2245
2246	dest = &tw->tw_v6_daddr;
2247	src = &tw->tw_v6_rcv_saddr;
2248	destp = ntohs(tw->tw_dport);
2249	srcp = ntohs(tw->tw_sport);
2250
2251	seq_printf(m: seq,
2252	fmt: "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2253	"%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2254	i,
2255	src->s6_addr32[0], src->s6_addr32[1],
2256	src->s6_addr32[2], src->s6_addr32[3], srcp,
2257	dest->s6_addr32[0], dest->s6_addr32[1],
2258	dest->s6_addr32[2], dest->s6_addr32[3], destp,
2259	tw->tw_substate, 0, 0,
2260	3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2261	refcount_read(r: &tw->tw_refcnt), tw);
2262	}
2263
2264	static int tcp6_seq_show(struct seq_file *seq, void *v)
2265	{
2266	struct tcp_iter_state *st;
2267	struct sock *sk = v;
2268
2269	if (v == SEQ_START_TOKEN) {
2270	seq_puts(m: seq,
2271	s: " sl "
2272	"local_address "
2273	"remote_address "
2274	"st tx_queue rx_queue tr tm->when retrnsmt"
2275	" uid timeout inode\n");
2276	goto out;
2277	}
2278	st = seq->private;
2279
2280	if (sk->sk_state == TCP_TIME_WAIT)
2281	get_timewait6_sock(seq, tw: v, i: st->num);
2282	else if (sk->sk_state == TCP_NEW_SYN_RECV)
2283	get_openreq6(seq, req: v, i: st->num);
2284	else
2285	get_tcp6_sock(seq, sp: v, i: st->num);
2286	out:
2287	return 0;
2288	}
2289
2290	static const struct seq_operations tcp6_seq_ops = {
2291	.show = tcp6_seq_show,
2292	.start = tcp_seq_start,
2293	.next = tcp_seq_next,
2294	.stop = tcp_seq_stop,
2295	};
2296
2297	static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2298	.family = AF_INET6,
2299	};
2300
2301	int __net_init tcp6_proc_init(struct net *net)
2302	{
2303	if (!proc_create_net_data(name: "tcp6", mode: 0444, parent: net->proc_net, ops: &tcp6_seq_ops,
2304	state_size: sizeof(struct tcp_iter_state), data: &tcp6_seq_afinfo))
2305	return -ENOMEM;
2306	return 0;
2307	}
2308
2309	void tcp6_proc_exit(struct net *net)
2310	{
2311	remove_proc_entry("tcp6", net->proc_net);
2312	}
2313	#endif
2314
2315	struct proto tcpv6_prot = {
2316	.name = "TCPv6",
2317	.owner = THIS_MODULE,
2318	.close = tcp_close,
2319	.pre_connect = tcp_v6_pre_connect,
2320	.connect = tcp_v6_connect,
2321	.disconnect = tcp_disconnect,
2322	.accept = inet_csk_accept,
2323	.ioctl = tcp_ioctl,
2324	.init = tcp_v6_init_sock,
2325	.destroy = tcp_v4_destroy_sock,
2326	.shutdown = tcp_shutdown,
2327	.setsockopt = tcp_setsockopt,
2328	.getsockopt = tcp_getsockopt,
2329	.bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
2330	.keepalive = tcp_set_keepalive,
2331	.recvmsg = tcp_recvmsg,
2332	.sendmsg = tcp_sendmsg,
2333	.splice_eof = tcp_splice_eof,
2334	.backlog_rcv = tcp_v6_do_rcv,
2335	.release_cb = tcp_release_cb,
2336	.hash = inet6_hash,
2337	.unhash = inet_unhash,
2338	.get_port = inet_csk_get_port,
2339	.put_port = inet_put_port,
2340	#ifdef CONFIG_BPF_SYSCALL
2341	.psock_update_sk_prot = tcp_bpf_update_proto,
2342	#endif
2343	.enter_memory_pressure = tcp_enter_memory_pressure,
2344	.leave_memory_pressure = tcp_leave_memory_pressure,
2345	.stream_memory_free = tcp_stream_memory_free,
2346	.sockets_allocated = &tcp_sockets_allocated,
2347
2348	.memory_allocated = &tcp_memory_allocated,
2349	.per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc,
2350
2351	.memory_pressure = &tcp_memory_pressure,
2352	.orphan_count = &tcp_orphan_count,
2353	.sysctl_mem = sysctl_tcp_mem,
2354	.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
2355	.sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
2356	.max_header = MAX_TCP_HEADER,
2357	.obj_size = sizeof(struct tcp6_sock),
2358	.ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
2359	.slab_flags = SLAB_TYPESAFE_BY_RCU,
2360	.twsk_prot = &tcp6_timewait_sock_ops,
2361	.rsk_prot = &tcp6_request_sock_ops,
2362	.h.hashinfo = NULL,
2363	.no_autobind = true,
2364	.diag_destroy = tcp_abort,
2365	};
2366	EXPORT_SYMBOL_GPL(tcpv6_prot);
2367
2368	static const struct inet6_protocol tcpv6_protocol = {
2369	.handler = tcp_v6_rcv,
2370	.err_handler = tcp_v6_err,
2371	.flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
2372	};
2373
2374	static struct inet_protosw tcpv6_protosw = {
2375	.type = SOCK_STREAM,
2376	.protocol = IPPROTO_TCP,
2377	.prot = &tcpv6_prot,
2378	.ops = &inet6_stream_ops,
2379	.flags = INET_PROTOSW_PERMANENT |
2380	INET_PROTOSW_ICSK,
2381	};
2382
2383	static int __net_init tcpv6_net_init(struct net *net)
2384	{
2385	return inet_ctl_sock_create(sk: &net->ipv6.tcp_sk, PF_INET6,
2386	type: SOCK_RAW, IPPROTO_TCP, net);
2387	}
2388
2389	static void __net_exit tcpv6_net_exit(struct net *net)
2390	{
2391	inet_ctl_sock_destroy(sk: net->ipv6.tcp_sk);
2392	}
2393
2394	static void __net_exit tcpv6_net_exit_batch(struct list_head *net_exit_list)
2395	{
2396	tcp_twsk_purge(net_exit_list, AF_INET6);
2397	}
2398
2399	static struct pernet_operations tcpv6_net_ops = {
2400	.init = tcpv6_net_init,
2401	.exit = tcpv6_net_exit,
2402	.exit_batch = tcpv6_net_exit_batch,
2403	};
2404
2405	int __init tcpv6_init(void)
2406	{
2407	int ret;
2408
2409	ret = inet6_add_protocol(prot: &tcpv6_protocol, IPPROTO_TCP);
2410	if (ret)
2411	goto out;
2412
2413	/* register inet6 protocol */
2414	ret = inet6_register_protosw(p: &tcpv6_protosw);
2415	if (ret)
2416	goto out_tcpv6_protocol;
2417
2418	ret = register_pernet_subsys(&tcpv6_net_ops);
2419	if (ret)
2420	goto out_tcpv6_protosw;
2421
2422	ret = mptcpv6_init();
2423	if (ret)
2424	goto out_tcpv6_pernet_subsys;
2425
2426	out:
2427	return ret;
2428
2429	out_tcpv6_pernet_subsys:
2430	unregister_pernet_subsys(&tcpv6_net_ops);
2431	out_tcpv6_protosw:
2432	inet6_unregister_protosw(p: &tcpv6_protosw);
2433	out_tcpv6_protocol:
2434	inet6_del_protocol(prot: &tcpv6_protocol, IPPROTO_TCP);
2435	goto out;
2436	}
2437
2438	void tcpv6_exit(void)
2439	{
2440	unregister_pernet_subsys(&tcpv6_net_ops);
2441	inet6_unregister_protosw(p: &tcpv6_protosw);
2442	inet6_del_protocol(prot: &tcpv6_protocol, IPPROTO_TCP);
2443	}
2444