1	// SPDX-License-Identifier: GPL-2.0
2	/* xfrm_iptfs: IPTFS encapsulation support
3	*
4	* April 21 2022, Christian Hopps <chopps@labn.net>
5	*
6	* Copyright (c) 2022, LabN Consulting, L.L.C.
7	*
8	*/
9
10	#include <linux/kernel.h>
11	#include <linux/icmpv6.h>
12	#include <linux/skbuff_ref.h>
13	#include <net/gro.h>
14	#include <net/icmp.h>
15	#include <net/ip6_route.h>
16	#include <net/inet_ecn.h>
17	#include <net/xfrm.h>
18
19	#include <crypto/aead.h>
20
21	#include "xfrm_inout.h"
22	#include "trace_iptfs.h"
23
24	/* IPTFS encap (header) values. */
25	#define IPTFS_SUBTYPE_BASIC 0
26	#define IPTFS_SUBTYPE_CC 1
27
28	/* ----------------------------------------------- */
29	/* IP-TFS default SA values (tunnel egress/dir-in) */
30	/* ----------------------------------------------- */
31
32	/**
33	* define IPTFS_DEFAULT_DROP_TIME_USECS - default drop time
34	*
35	* The default IPTFS drop time in microseconds. The drop time is the amount of
36	* time before a missing out-of-order IPTFS tunnel packet is considered lost.
37	* See also the reorder window.
38	*
39	* Default 1s.
40	*/
41	#define IPTFS_DEFAULT_DROP_TIME_USECS 1000000
42
43	/**
44	* define IPTFS_DEFAULT_REORDER_WINDOW - default reorder window size
45	*
46	* The default IPTFS reorder window size. The reorder window size dictates the
47	* maximum number of IPTFS tunnel packets in a sequence that may arrive out of
48	* order.
49	*
50	* Default 3. (tcp folks suggested)
51	*/
52	#define IPTFS_DEFAULT_REORDER_WINDOW 3
53
54	/* ------------------------------------------------ */
55	/* IPTFS default SA values (tunnel ingress/dir-out) */
56	/* ------------------------------------------------ */
57
58	/**
59	* define IPTFS_DEFAULT_INIT_DELAY_USECS - default initial output delay
60	*
61	* The initial output delay is the amount of time prior to servicing the output
62	* queue after queueing the first packet on said queue. This applies anytime the
63	* output queue was previously empty.
64	*
65	* Default 0.
66	*/
67	#define IPTFS_DEFAULT_INIT_DELAY_USECS 0
68
69	/**
70	* define IPTFS_DEFAULT_MAX_QUEUE_SIZE - default max output queue size.
71	*
72	* The default IPTFS max output queue size in octets. The output queue is where
73	* received packets destined for output over an IPTFS tunnel are stored prior to
74	* being output in aggregated/fragmented form over the IPTFS tunnel.
75	*
76	* Default 1M.
77	*/
78	#define IPTFS_DEFAULT_MAX_QUEUE_SIZE (1024 * 10240)
79
80	/* Assumed: skb->head is cache aligned.
81	*
82	* L2 Header resv: Arrange for cacheline to start at skb->data - 16 to keep the
83	* to-be-pushed L2 header in the same cacheline as resulting `skb->data` (i.e.,
84	* the L3 header). If cacheline size is > 64 then skb->data + pushed L2 will all
85	* be in a single cacheline if we simply reserve 64 bytes.
86	*
87	* L3 Header resv: For L3+L2 headers (i.e., skb->data points at the IPTFS payload)
88	* we want `skb->data` to be cacheline aligned and all pushed L2L3 headers will
89	* be in their own cacheline[s]. 128 works for cachelins up to 128 bytes, for
90	* any larger cacheline sizes the pushed headers will simply share the cacheline
91	* with the start of the IPTFS payload (skb->data).
92	*/
93	#define XFRM_IPTFS_MIN_L3HEADROOM 128
94	#define XFRM_IPTFS_MIN_L2HEADROOM (L1_CACHE_BYTES > 64 ? 64 : 64 + 16)
95
96	/* Min to try to share outer iptfs skb data vs copying into new skb */
97	#define IPTFS_PKT_SHARE_MIN 129
98
99	#define NSECS_IN_USEC 1000
100
101	#define IPTFS_HRTIMER_MODE HRTIMER_MODE_REL_SOFT
102
103	/**
104	* struct xfrm_iptfs_config - configuration for the IPTFS tunnel.
105	* @pkt_size: size of the outer IP packet. 0 to use interface and MTU discovery,
106	* otherwise the user specified value.
107	* @max_queue_size: The maximum number of octets allowed to be queued to be sent
108	* over the IPTFS SA. The queue size is measured as the size of all the
109	* packets enqueued.
110	* @reorder_win_size: the number slots in the reorder window, thus the number of
111	* packets that may arrive out of order.
112	* @dont_frag: true to inhibit fragmenting across IPTFS outer packets.
113	*/
114	struct xfrm_iptfs_config {
115	u32 pkt_size; /* outer_packet_size or 0 */
116	u32 max_queue_size; /* octets */
117	u16 reorder_win_size;
118	u8 dont_frag : 1;
119	};
120
121	struct skb_wseq {
122	struct sk_buff *skb;
123	u64 drop_time;
124	};
125
126	/**
127	* struct xfrm_iptfs_data - mode specific xfrm state.
128	* @cfg: IPTFS tunnel config.
129	* @x: owning SA (xfrm_state).
130	* @queue: queued user packets to send.
131	* @queue_size: number of octets on queue (sum of packet sizes).
132	* @ecn_queue_size: octets above with ECN mark.
133	* @init_delay_ns: nanoseconds to wait to send initial IPTFS packet.
134	* @iptfs_timer: output timer.
135	* @iptfs_settime: time the output timer was set.
136	* @payload_mtu: max payload size.
137	* @w_seq_set: true after first seq received.
138	* @w_wantseq: waiting for this seq number as next to process (in order).
139	* @w_saved: the saved buf array (reorder window).
140	* @w_savedlen: the saved len (not size).
141	* @drop_lock: lock to protect reorder queue.
142	* @drop_timer: timer for considering next packet lost.
143	* @drop_time_ns: timer intervan in nanoseconds.
144	* @ra_newskb: new pkt being reassembled.
145	* @ra_wantseq: expected next sequence for reassembly.
146	* @ra_runt: last pkt bytes from very end of last skb.
147	* @ra_runtlen: size of ra_runt.
148	*/
149	struct xfrm_iptfs_data {
150	struct xfrm_iptfs_config cfg;
151
152	/* Ingress User Input */
153	struct xfrm_state *x; /* owning state */
154	struct sk_buff_head queue; /* output queue */
155
156	u32 queue_size; /* octets */
157	u32 ecn_queue_size; /* octets above which ECN mark */
158	u64 init_delay_ns; /* nanoseconds */
159	struct hrtimer iptfs_timer; /* output timer */
160	time64_t iptfs_settime; /* time timer was set */
161	u32 payload_mtu; /* max payload size */
162
163	/* Tunnel input reordering */
164	bool w_seq_set; /* true after first seq received */
165	u64 w_wantseq; /* expected next sequence */
166	struct skb_wseq *w_saved; /* the saved buf array */
167	u32 w_savedlen; /* the saved len (not size) */
168	spinlock_t drop_lock;
169	struct hrtimer drop_timer;
170	u64 drop_time_ns;
171
172	/* Tunnel input reassembly */
173	struct sk_buff *ra_newskb; /* new pkt being reassembled */
174	u64 ra_wantseq; /* expected next sequence */
175	u8 ra_runt[6]; /* last pkt bytes from last skb */
176	u8 ra_runtlen; /* count of ra_runt */
177	};
178
179	static u32 __iptfs_get_inner_mtu(struct xfrm_state *x, int outer_mtu);
180	static enum hrtimer_restart iptfs_delay_timer(struct hrtimer *me);
181	static enum hrtimer_restart iptfs_drop_timer(struct hrtimer *me);
182
183	/* ================= */
184	/* Utility Functions */
185	/* ================= */
186
187	#ifdef TRACEPOINTS_ENABLED
188	static u32 __trace_ip_proto(struct iphdr *iph)
189	{
190	if (iph->version == 4)
191	return iph->protocol;
192	return ((struct ipv6hdr *)iph)->nexthdr;
193	}
194
195	static u32 __trace_ip_proto_seq(struct iphdr *iph)
196	{
197	void *nexthdr;
198	u32 protocol = 0;
199
200	if (iph->version == 4) {
201	nexthdr = (void *)(iph + 1);
202	protocol = iph->protocol;
203	} else if (iph->version == 6) {
204	nexthdr = (void *)(((struct ipv6hdr *)(iph)) + 1);
205	protocol = ((struct ipv6hdr *)(iph))->nexthdr;
206	}
207	switch (protocol) {
208	case IPPROTO_ICMP:
209	return ntohs(((struct icmphdr *)nexthdr)->un.echo.sequence);
210	case IPPROTO_ICMPV6:
211	return ntohs(((struct icmp6hdr *)nexthdr)->icmp6_sequence);
212	case IPPROTO_TCP:
213	return ntohl(((struct tcphdr *)nexthdr)->seq);
214	case IPPROTO_UDP:
215	return ntohs(((struct udphdr *)nexthdr)->source);
216	default:
217	return 0;
218	}
219	}
220	#endif /*TRACEPOINTS_ENABLED*/
221
222	static u64 __esp_seq(struct sk_buff *skb)
223	{
224	u64 seq = ntohl(XFRM_SKB_CB(skb)->seq.input.low);
225
226	return seq | (u64)ntohl(XFRM_SKB_CB(skb)->seq.input.hi) << 32;
227	}
228
229	/* ======================= */
230	/* IPTFS SK_BUFF Functions */
231	/* ======================= */
232
233	/**
234	* iptfs_alloc_skb() - Allocate a new `skb`.
235	* @tpl: the skb to copy required meta-data from.
236	* @len: the linear length of the head data, zero is fine.
237	* @l3resv: true if skb reserve needs to support pushing L3 headers
238	*
239	* A new `skb` is allocated and required meta-data is copied from `tpl`, the
240	* head data is sized to `len` + reserved space set according to the @l3resv
241	* boolean.
242	*
243	* When @l3resv is false, resv is XFRM_IPTFS_MIN_L2HEADROOM which arranges for
244	* `skb->data - 16` which is a good guess for good cache alignment (placing the
245	* to be pushed L2 header at the start of a cacheline.
246	*
247	* Otherwise, @l3resv is true and resv is set to the correct reserved space for
248	* dst->dev plus the calculated L3 overhead for the xfrm dst or
249	* XFRM_IPTFS_MIN_L3HEADROOM whichever is larger. This is then cache aligned so
250	* that all the headers will commonly fall in a cacheline when possible.
251	*
252	* l3resv=true is used on tunnel ingress (tx), because we need to reserve for
253	* the new IPTFS packet (i.e., L2+L3 headers). On tunnel egress (rx) the data
254	* being copied into the skb includes the user L3 headers already so we only
255	* need to reserve for L2.
256	*
257	* Return: the new skb or NULL.
258	*/
259	static struct sk_buff *iptfs_alloc_skb(struct sk_buff *tpl, u32 len, bool l3resv)
260	{
261	struct sk_buff *skb;
262	u32 resv;
263
264	if (!l3resv) {
265	resv = XFRM_IPTFS_MIN_L2HEADROOM;
266	} else {
267	struct dst_entry *dst = skb_dst(skb: tpl);
268
269	resv = LL_RESERVED_SPACE(dst->dev) + dst->header_len;
270	resv = max(resv, XFRM_IPTFS_MIN_L3HEADROOM);
271	resv = L1_CACHE_ALIGN(resv);
272	}
273
274	skb = alloc_skb(size: len + resv, GFP_ATOMIC | __GFP_NOWARN);
275	if (!skb)
276	return NULL;
277
278	skb_reserve(skb, len: resv);
279
280	if (!l3resv) {
281	/* xfrm_input resume needs dev and xfrm ext from tunnel pkt */
282	skb->dev = tpl->dev;
283	__skb_ext_copy(dst: skb, src: tpl);
284	}
285
286	/* dropped by xfrm_input, used by xfrm_output */
287	skb_dst_copy(nskb: skb, oskb: tpl);
288
289	return skb;
290	}
291
292	/**
293	* iptfs_skb_head_to_frag() - initialize a skb_frag_t based on skb head data
294	* @skb: skb with the head data
295	* @frag: frag to initialize
296	*/
297	static void iptfs_skb_head_to_frag(const struct sk_buff *skb, skb_frag_t *frag)
298	{
299	struct page *page = virt_to_head_page(x: skb->data);
300	unsigned char *addr = (unsigned char *)page_address(page);
301
302	skb_frag_fill_page_desc(frag, page, off: skb->data - addr, size: skb_headlen(skb));
303	}
304
305	/**
306	* struct iptfs_skb_frag_walk - use to track a walk through fragments
307	* @fragi: current fragment index
308	* @past: length of data in fragments before @fragi
309	* @total: length of data in all fragments
310	* @nr_frags: number of fragments present in array
311	* @initial_offset: the value passed in to skb_prepare_frag_walk()
312	* @frags: the page fragments inc. room for head page
313	* @pp_recycle: copy of skb->pp_recycle
314	*/
315	struct iptfs_skb_frag_walk {
316	u32 fragi;
317	u32 past;
318	u32 total;
319	u32 nr_frags;
320	u32 initial_offset;
321	skb_frag_t frags[MAX_SKB_FRAGS + 1];
322	bool pp_recycle;
323	};
324
325	/**
326	* iptfs_skb_prepare_frag_walk() - initialize a frag walk over an skb.
327	* @skb: the skb to walk.
328	* @initial_offset: start the walk @initial_offset into the skb.
329	* @walk: the walk to initialize
330	*
331	* Future calls to skb_add_frags() will expect the @offset value to be at
332	* least @initial_offset large.
333	*/
334	static void iptfs_skb_prepare_frag_walk(struct sk_buff *skb, u32 initial_offset,
335	struct iptfs_skb_frag_walk *walk)
336	{
337	struct skb_shared_info *shinfo = skb_shinfo(skb);
338	skb_frag_t *frag, *from;
339	u32 i;
340
341	walk->initial_offset = initial_offset;
342	walk->fragi = 0;
343	walk->past = 0;
344	walk->total = 0;
345	walk->nr_frags = 0;
346	walk->pp_recycle = skb->pp_recycle;
347
348	if (skb->head_frag) {
349	if (initial_offset >= skb_headlen(skb)) {
350	initial_offset -= skb_headlen(skb);
351	} else {
352	frag = &walk->frags[walk->nr_frags++];
353	iptfs_skb_head_to_frag(skb, frag);
354	frag->offset += initial_offset;
355	frag->len -= initial_offset;
356	walk->total += frag->len;
357	initial_offset = 0;
358	}
359	} else {
360	initial_offset -= skb_headlen(skb);
361	}
362
363	for (i = 0; i < shinfo->nr_frags; i++) {
364	from = &shinfo->frags[i];
365	if (initial_offset >= from->len) {
366	initial_offset -= from->len;
367	continue;
368	}
369	frag = &walk->frags[walk->nr_frags++];
370	*frag = *from;
371	if (initial_offset) {
372	frag->offset += initial_offset;
373	frag->len -= initial_offset;
374	initial_offset = 0;
375	}
376	walk->total += frag->len;
377	}
378	}
379
380	static u32 iptfs_skb_reset_frag_walk(struct iptfs_skb_frag_walk *walk,
381	u32 offset)
382	{
383	/* Adjust offset to refer to internal walk values */
384	offset -= walk->initial_offset;
385
386	/* Get to the correct fragment for offset */
387	while (offset < walk->past) {
388	walk->past -= walk->frags[--walk->fragi].len;
389	if (offset >= walk->past)
390	break;
391	}
392	while (offset >= walk->past + walk->frags[walk->fragi].len)
393	walk->past += walk->frags[walk->fragi++].len;
394
395	/* offset now relative to this current frag */
396	offset -= walk->past;
397	return offset;
398	}
399
400	/**
401	* iptfs_skb_can_add_frags() - check if ok to add frags from walk to skb
402	* @skb: skb to check for adding frags to
403	* @walk: the walk that will be used as source for frags.
404	* @offset: offset from beginning of original skb to start from.
405	* @len: amount of data to add frag references to in @skb.
406	*
407	* Return: true if ok to add frags.
408	*/
409	static bool iptfs_skb_can_add_frags(const struct sk_buff *skb,
410	struct iptfs_skb_frag_walk *walk,
411	u32 offset, u32 len)
412	{
413	struct skb_shared_info *shinfo = skb_shinfo(skb);
414	u32 fragi, nr_frags, fraglen;
415
416	if (skb_has_frag_list(skb) || skb->pp_recycle != walk->pp_recycle)
417	return false;
418
419	/* Make offset relative to current frag after setting that */
420	offset = iptfs_skb_reset_frag_walk(walk, offset);
421
422	/* Verify we have array space for the fragments we need to add */
423	fragi = walk->fragi;
424	nr_frags = shinfo->nr_frags;
425	while (len && fragi < walk->nr_frags) {
426	skb_frag_t *frag = &walk->frags[fragi];
427
428	fraglen = frag->len;
429	if (offset) {
430	fraglen -= offset;
431	offset = 0;
432	}
433	if (++nr_frags > MAX_SKB_FRAGS)
434	return false;
435	if (len <= fraglen)
436	return true;
437	len -= fraglen;
438	fragi++;
439	}
440	/* We may not copy all @len but what we have will fit. */
441	return true;
442	}
443
444	/**
445	* iptfs_skb_add_frags() - add a range of fragment references into an skb
446	* @skb: skb to add references into
447	* @walk: the walk to add referenced fragments from.
448	* @offset: offset from beginning of original skb to start from.
449	* @len: amount of data to add frag references to in @skb.
450	*
451	* iptfs_skb_can_add_frags() should be called before this function to verify
452	* that the destination @skb is compatible with the walk and has space in the
453	* array for the to be added frag references.
454	*
455	* Return: The number of bytes not added to @skb b/c we reached the end of the
456	* walk before adding all of @len.
457	*/
458	static int iptfs_skb_add_frags(struct sk_buff *skb,
459	struct iptfs_skb_frag_walk *walk, u32 offset,
460	u32 len)
461	{
462	struct skb_shared_info *shinfo = skb_shinfo(skb);
463	u32 fraglen;
464
465	if (!walk->nr_frags || offset >= walk->total + walk->initial_offset)
466	return len;
467
468	/* make offset relative to current frag after setting that */
469	offset = iptfs_skb_reset_frag_walk(walk, offset);
470
471	while (len && walk->fragi < walk->nr_frags) {
472	skb_frag_t *frag = &walk->frags[walk->fragi];
473	skb_frag_t *tofrag = &shinfo->frags[shinfo->nr_frags];
474
475	*tofrag = *frag;
476	if (offset) {
477	tofrag->offset += offset;
478	tofrag->len -= offset;
479	offset = 0;
480	}
481	__skb_frag_ref(frag: tofrag);
482	shinfo->nr_frags++;
483
484	/* see if we are done */
485	fraglen = tofrag->len;
486	if (len < fraglen) {
487	tofrag->len = len;
488	skb->len += len;
489	skb->data_len += len;
490	return 0;
491	}
492	/* advance to next source fragment */
493	len -= fraglen; /* careful, use dst bv_len */
494	skb->len += fraglen; /* careful, " " " */
495	skb->data_len += fraglen; /* careful, " " " */
496	walk->past += frag->len; /* careful, use src bv_len */
497	walk->fragi++;
498	}
499	return len;
500	}
501
502	/* ================================== */
503	/* IPTFS Trace Event Definitions */
504	/* ================================== */
505
506	#define CREATE_TRACE_POINTS
507	#include "trace_iptfs.h"
508
509	/* ================================== */
510	/* IPTFS Receiving (egress) Functions */
511	/* ================================== */
512
513	/**
514	* iptfs_pskb_add_frags() - Create and add frags into a new sk_buff.
515	* @tpl: template to create new skb from.
516	* @walk: The source for fragments to add.
517	* @off: The offset into @walk to add frags from, also used with @st and
518	* @copy_len.
519	* @len: The length of data to add covering frags from @walk into @skb.
520	* This must be <= @skblen.
521	* @st: The sequence state to copy from into the new head skb.
522	* @copy_len: Copy @copy_len bytes from @st at offset @off into the new skb
523	* linear space.
524	*
525	* Create a new sk_buff `skb` using the template @tpl. Copy @copy_len bytes from
526	* @st into the new skb linear space, and then add shared fragments from the
527	* frag walk for the remaining @len of data (i.e., @len - @copy_len bytes).
528	*
529	* Return: The newly allocated sk_buff `skb` or NULL if an error occurs.
530	*/
531	static struct sk_buff *
532	iptfs_pskb_add_frags(struct sk_buff *tpl, struct iptfs_skb_frag_walk *walk,
533	u32 off, u32 len, struct skb_seq_state *st, u32 copy_len)
534	{
535	struct sk_buff *skb;
536
537	skb = iptfs_alloc_skb(tpl, len: copy_len, l3resv: false);
538	if (!skb)
539	return NULL;
540
541	/* this should not normally be happening */
542	if (!iptfs_skb_can_add_frags(skb, walk, offset: off + copy_len,
543	len: len - copy_len)) {
544	kfree_skb(skb);
545	return NULL;
546	}
547
548	if (copy_len &&
549	skb_copy_seq_read(st, offset: off, to: skb_put(skb, len: copy_len), len: copy_len)) {
550	XFRM_INC_STATS(dev_net(st->root_skb->dev),
551	LINUX_MIB_XFRMINERROR);
552	kfree_skb(skb);
553	return NULL;
554	}
555
556	iptfs_skb_add_frags(skb, walk, offset: off + copy_len, len: len - copy_len);
557	return skb;
558	}
559
560	/**
561	* iptfs_pskb_extract_seq() - Create and load data into a new sk_buff.
562	* @skblen: the total data size for `skb`.
563	* @st: The source for the rest of the data to copy into `skb`.
564	* @off: The offset into @st to copy data from.
565	* @len: The length of data to copy from @st into `skb`. This must be <=
566	* @skblen.
567	*
568	* Create a new sk_buff `skb` with @skblen of packet data space. If non-zero,
569	* copy @rlen bytes of @runt into `skb`. Then using seq functions copy @len
570	* bytes from @st into `skb` starting from @off.
571	*
572	* It is an error for @len to be greater than the amount of data left in @st.
573	*
574	* Return: The newly allocated sk_buff `skb` or NULL if an error occurs.
575	*/
576	static struct sk_buff *
577	iptfs_pskb_extract_seq(u32 skblen, struct skb_seq_state *st, u32 off, int len)
578	{
579	struct sk_buff *skb = iptfs_alloc_skb(tpl: st->root_skb, len: skblen, l3resv: false);
580
581	if (!skb)
582	return NULL;
583	if (skb_copy_seq_read(st, offset: off, to: skb_put(skb, len), len)) {
584	XFRM_INC_STATS(dev_net(st->root_skb->dev), LINUX_MIB_XFRMINERROR);
585	kfree_skb(skb);
586	return NULL;
587	}
588	return skb;
589	}
590
591	/**
592	* iptfs_input_save_runt() - save data in xtfs runt space.
593	* @xtfs: xtfs state
594	* @seq: the current sequence
595	* @buf: packet data
596	* @len: length of packet data
597	*
598	* Save the small (`len`) start of a fragmented packet in `buf` in the xtfs data
599	* runt space.
600	*/
601	static void iptfs_input_save_runt(struct xfrm_iptfs_data *xtfs, u64 seq,
602	u8 *buf, int len)
603	{
604	memcpy(xtfs->ra_runt, buf, len);
605
606	xtfs->ra_runtlen = len;
607	xtfs->ra_wantseq = seq + 1;
608	}
609
610	/**
611	* __iptfs_iphlen() - return the v4/v6 header length using packet data.
612	* @data: pointer at octet with version nibble
613	*
614	* The version data has been checked to be valid (i.e., either 4 or 6).
615	*
616	* Return: the IP header size based on the IP version.
617	*/
618	static u32 __iptfs_iphlen(u8 *data)
619	{
620	struct iphdr *iph = (struct iphdr *)data;
621
622	if (iph->version == 0x4)
623	return sizeof(*iph);
624	return sizeof(struct ipv6hdr);
625	}
626
627	/**
628	* __iptfs_iplen() - return the v4/v6 length using packet data.
629	* @data: pointer to ip (v4/v6) packet header
630	*
631	* Grab the IPv4 or IPv6 length value in the start of the inner packet header
632	* pointed to by `data`. Assumes data len is enough for the length field only.
633	*
634	* The version data has been checked to be valid (i.e., either 4 or 6).
635	*
636	* Return: the length value.
637	*/
638	static u32 __iptfs_iplen(u8 *data)
639	{
640	struct iphdr *iph = (struct iphdr *)data;
641
642	if (iph->version == 0x4)
643	return ntohs(iph->tot_len);
644	return ntohs(((struct ipv6hdr *)iph)->payload_len) +
645	sizeof(struct ipv6hdr);
646	}
647
648	/**
649	* iptfs_complete_inner_skb() - finish preparing the inner packet for gro recv.
650	* @x: xfrm state
651	* @skb: the inner packet
652	*
653	* Finish the standard xfrm processing on the inner packet prior to sending back
654	* through gro_cells_receive. We do this separately b/c we are building a list
655	* of packets in the hopes that one day a list will be taken by
656	* xfrm_input.
657	*/
658	static void iptfs_complete_inner_skb(struct xfrm_state *x, struct sk_buff *skb)
659	{
660	skb_reset_network_header(skb);
661
662	/* The packet is going back through gro_cells_receive no need to
663	* set this.
664	*/
665	skb_reset_transport_header(skb);
666
667	/* Packet already has checksum value set. */
668	skb->ip_summed = CHECKSUM_NONE;
669
670	/* Our skb will contain the header data copied when this outer packet
671	* which contained the start of this inner packet. This is true
672	* when we allocate a new skb as well as when we reuse the existing skb.
673	*/
674	if (ip_hdr(skb)->version == 0x4) {
675	struct iphdr *iph = ip_hdr(skb);
676
677	if (x->props.flags & XFRM_STATE_DECAP_DSCP)
678	ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, inner: iph);
679	if (!(x->props.flags & XFRM_STATE_NOECN))
680	if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
681	IP_ECN_set_ce(iph);
682
683	skb->protocol = htons(ETH_P_IP);
684	} else {
685	struct ipv6hdr *iph = ipv6_hdr(skb);
686
687	if (x->props.flags & XFRM_STATE_DECAP_DSCP)
688	ipv6_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, inner: iph);
689	if (!(x->props.flags & XFRM_STATE_NOECN))
690	if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
691	IP6_ECN_set_ce(skb, iph);
692
693	skb->protocol = htons(ETH_P_IPV6);
694	}
695	}
696
697	static void __iptfs_reassem_done(struct xfrm_iptfs_data *xtfs, bool free)
698	{
699	assert_spin_locked(&xtfs->drop_lock);
700
701	/* We don't care if it works locking takes care of things */
702	hrtimer_try_to_cancel(timer: &xtfs->drop_timer);
703	if (free)
704	kfree_skb(skb: xtfs->ra_newskb);
705	xtfs->ra_newskb = NULL;
706	}
707
708	/**
709	* iptfs_reassem_abort() - In-progress packet is aborted free the state.
710	* @xtfs: xtfs state
711	*/
712	static void iptfs_reassem_abort(struct xfrm_iptfs_data *xtfs)
713	{
714	__iptfs_reassem_done(xtfs, free: true);
715	}
716
717	/**
718	* iptfs_reassem_done() - In-progress packet is complete, clear the state.
719	* @xtfs: xtfs state
720	*/
721	static void iptfs_reassem_done(struct xfrm_iptfs_data *xtfs)
722	{
723	__iptfs_reassem_done(xtfs, free: false);
724	}
725
726	/**
727	* iptfs_reassem_cont() - Continue the reassembly of an inner packets.
728	* @xtfs: xtfs state
729	* @seq: sequence of current packet
730	* @st: seq read stat for current packet
731	* @skb: current packet
732	* @data: offset into sequential packet data
733	* @blkoff: packet blkoff value
734	* @list: list of skbs to enqueue completed packet on
735	*
736	* Process an IPTFS payload that has a non-zero `blkoff` or when we are
737	* expecting the continuation b/c we have a runt or in-progress packet.
738	*
739	* Return: the new data offset to continue processing from.
740	*/
741	static u32 iptfs_reassem_cont(struct xfrm_iptfs_data *xtfs, u64 seq,
742	struct skb_seq_state *st, struct sk_buff *skb,
743	u32 data, u32 blkoff, struct list_head *list)
744	{
745	struct iptfs_skb_frag_walk _fragwalk;
746	struct iptfs_skb_frag_walk *fragwalk = NULL;
747	struct sk_buff *newskb = xtfs->ra_newskb;
748	u32 remaining = skb->len - data;
749	u32 runtlen = xtfs->ra_runtlen;
750	u32 copylen, fraglen, ipremain, iphlen, iphremain, rrem;
751
752	/* Handle packet fragment we aren't expecting */
753	if (!runtlen && !xtfs->ra_newskb)
754	return data + min(blkoff, remaining);
755
756	/* Important to remember that input to this function is an ordered
757	* packet stream (unless the user disabled the reorder window). Thus if
758	* we are waiting for, and expecting the next packet so we can continue
759	* assembly, a newer sequence number indicates older ones are not coming
760	* (or if they do should be ignored). Technically we can receive older
761	* ones when the reorder window is disabled; however, the user should
762	* have disabled fragmentation in this case, and regardless we don't
763	* deal with it.
764	*
765	* blkoff could be zero if the stream is messed up (or it's an all pad
766	* insertion) be careful to handle that case in each of the below
767	*/
768
769	/* Too old case: This can happen when the reorder window is disabled so
770	* ordering isn't actually guaranteed.
771	*/
772	if (seq < xtfs->ra_wantseq)
773	return data + remaining;
774
775	/* Too new case: We missed what we wanted cleanup. */
776	if (seq > xtfs->ra_wantseq) {
777	XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINIPTFSERROR);
778	goto abandon;
779	}
780
781	if (blkoff == 0) {
782	if ((*skb->data & 0xF0) != 0) {
783	XFRM_INC_STATS(xs_net(xtfs->x),
784	LINUX_MIB_XFRMINIPTFSERROR);
785	goto abandon;
786	}
787	/* Handle all pad case, advance expected sequence number.
788	* (RFC 9347 S2.2.3)
789	*/
790	xtfs->ra_wantseq++;
791	/* will end parsing */
792	return data + remaining;
793	}
794
795	if (runtlen) {
796	/* Regardless of what happens we're done with the runt */
797	xtfs->ra_runtlen = 0;
798
799	/* The start of this inner packet was at the very end of the last
800	* iptfs payload which didn't include enough for the ip header
801	* length field. We must have *at least* that now.
802	*/
803	rrem = sizeof(xtfs->ra_runt) - runtlen;
804	if (remaining < rrem || blkoff < rrem) {
805	XFRM_INC_STATS(xs_net(xtfs->x),
806	LINUX_MIB_XFRMINIPTFSERROR);
807	goto abandon;
808	}
809
810	/* fill in the runt data */
811	if (skb_copy_seq_read(st, offset: data, to: &xtfs->ra_runt[runtlen],
812	len: rrem)) {
813	XFRM_INC_STATS(xs_net(xtfs->x),
814	LINUX_MIB_XFRMINBUFFERERROR);
815	goto abandon;
816	}
817
818	/* We have enough data to get the ip length value now,
819	* allocate an in progress skb
820	*/
821	ipremain = __iptfs_iplen(data: xtfs->ra_runt);
822	if (ipremain < sizeof(xtfs->ra_runt)) {
823	/* length has to be at least runtsize large */
824	XFRM_INC_STATS(xs_net(xtfs->x),
825	LINUX_MIB_XFRMINIPTFSERROR);
826	goto abandon;
827	}
828
829	/* For the runt case we don't attempt sharing currently. NOTE:
830	* Currently, this IPTFS implementation will not create runts.
831	*/
832
833	newskb = iptfs_alloc_skb(tpl: skb, len: ipremain, l3resv: false);
834	if (!newskb) {
835	XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINERROR);
836	goto abandon;
837	}
838	xtfs->ra_newskb = newskb;
839
840	/* Copy the runt data into the buffer, but leave data
841	* pointers the same as normal non-runt case. The extra `rrem`
842	* recopied bytes are basically cacheline free. Allows using
843	* same logic below to complete.
844	*/
845	memcpy(skb_put(newskb, runtlen), xtfs->ra_runt,
846	sizeof(xtfs->ra_runt));
847	}
848
849	/* Continue reassembling the packet */
850	ipremain = __iptfs_iplen(data: newskb->data);
851	iphlen = __iptfs_iphlen(data: newskb->data);
852
853	ipremain -= newskb->len;
854	if (blkoff < ipremain) {
855	/* Corrupt data, we don't have enough to complete the packet */
856	XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINIPTFSERROR);
857	goto abandon;
858	}
859
860	/* We want the IP header in linear space */
861	if (newskb->len < iphlen) {
862	iphremain = iphlen - newskb->len;
863	if (blkoff < iphremain) {
864	XFRM_INC_STATS(xs_net(xtfs->x),
865	LINUX_MIB_XFRMINIPTFSERROR);
866	goto abandon;
867	}
868	fraglen = min(blkoff, remaining);
869	copylen = min(fraglen, iphremain);
870	if (skb_copy_seq_read(st, offset: data, to: skb_put(skb: newskb, len: copylen),
871	len: copylen)) {
872	XFRM_INC_STATS(xs_net(xtfs->x),
873	LINUX_MIB_XFRMINBUFFERERROR);
874	goto abandon;
875	}
876	/* this is a silly condition that might occur anyway */
877	if (copylen < iphremain) {
878	xtfs->ra_wantseq++;
879	return data + fraglen;
880	}
881	/* update data and things derived from it */
882	data += copylen;
883	blkoff -= copylen;
884	remaining -= copylen;
885	ipremain -= copylen;
886	}
887
888	fraglen = min(blkoff, remaining);
889	copylen = min(fraglen, ipremain);
890
891	/* If we may have the opportunity to share prepare a fragwalk. */
892	if (!skb_has_frag_list(skb) && !skb_has_frag_list(skb: newskb) &&
893	(skb->head_frag || skb->len == skb->data_len) &&
894	skb->pp_recycle == newskb->pp_recycle) {
895	fragwalk = &_fragwalk;
896	iptfs_skb_prepare_frag_walk(skb, initial_offset: data, walk: fragwalk);
897	}
898
899	/* Try share then copy. */
900	if (fragwalk &&
901	iptfs_skb_can_add_frags(skb: newskb, walk: fragwalk, offset: data, len: copylen)) {
902	iptfs_skb_add_frags(skb: newskb, walk: fragwalk, offset: data, len: copylen);
903	} else {
904	/* copy fragment data into newskb */
905	if (skb_copy_seq_read(st, offset: data, to: skb_put(skb: newskb, len: copylen),
906	len: copylen)) {
907	XFRM_INC_STATS(xs_net(xtfs->x),
908	LINUX_MIB_XFRMINBUFFERERROR);
909	goto abandon;
910	}
911	}
912
913	if (copylen < ipremain) {
914	xtfs->ra_wantseq++;
915	} else {
916	/* We are done with packet reassembly! */
917	iptfs_reassem_done(xtfs);
918	iptfs_complete_inner_skb(x: xtfs->x, skb: newskb);
919	list_add_tail(new: &newskb->list, head: list);
920	}
921
922	/* will continue on to new data block or end */
923	return data + fraglen;
924
925	abandon:
926	if (xtfs->ra_newskb) {
927	iptfs_reassem_abort(xtfs);
928	} else {
929	xtfs->ra_runtlen = 0;
930	xtfs->ra_wantseq = 0;
931	}
932	/* skip past fragment, maybe to end */
933	return data + min(blkoff, remaining);
934	}
935
936	static bool __input_process_payload(struct xfrm_state *x, u32 data,
937	struct skb_seq_state *skbseq,
938	struct list_head *sublist)
939	{
940	u8 hbytes[sizeof(struct ipv6hdr)];
941	struct iptfs_skb_frag_walk _fragwalk;
942	struct iptfs_skb_frag_walk *fragwalk = NULL;
943	struct sk_buff *defer, *first_skb, *next, *skb;
944	const unsigned char *old_mac;
945	struct xfrm_iptfs_data *xtfs;
946	struct iphdr *iph;
947	struct net *net;
948	u32 first_iplen, iphlen, iplen, remaining, tail;
949	u32 capturelen;
950	u64 seq;
951
952	xtfs = x->mode_data;
953	net = xs_net(x);
954	skb = skbseq->root_skb;
955	first_skb = NULL;
956	defer = NULL;
957
958	seq = __esp_seq(skb);
959
960	/* Save the old mac header if set */
961	old_mac = skb_mac_header_was_set(skb) ? skb_mac_header(skb) : NULL;
962
963	/* New packets */
964
965	tail = skb->len;
966	while (data < tail) {
967	__be16 protocol = 0;
968
969	/* Gather information on the next data block.
970	* `data` points to the start of the data block.
971	*/
972	remaining = tail - data;
973
974	/* try and copy enough bytes to read length from ipv4/ipv6 */
975	iphlen = min_t(u32, remaining, 6);
976	if (skb_copy_seq_read(st: skbseq, offset: data, to: hbytes, len: iphlen)) {
977	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
978	goto done;
979	}
980
981	iph = (struct iphdr *)hbytes;
982	if (iph->version == 0x4) {
983	/* must have at least tot_len field present */
984	if (remaining < 4) {
985	/* save the bytes we have, advance data and exit */
986	iptfs_input_save_runt(xtfs, seq, buf: hbytes,
987	len: remaining);
988	data += remaining;
989	break;
990	}
991
992	iplen = be16_to_cpu(iph->tot_len);
993	iphlen = iph->ihl << 2;
994	protocol = cpu_to_be16(ETH_P_IP);
995	XFRM_MODE_SKB_CB(skbseq->root_skb)->tos = iph->tos;
996	} else if (iph->version == 0x6) {
997	/* must have at least payload_len field present */
998	if (remaining < 6) {
999	/* save the bytes we have, advance data and exit */
1000	iptfs_input_save_runt(xtfs, seq, buf: hbytes,
1001	len: remaining);
1002	data += remaining;
1003	break;
1004	}
1005
1006	iplen = be16_to_cpu(((struct ipv6hdr *)hbytes)->payload_len);
1007	iplen += sizeof(struct ipv6hdr);
1008	iphlen = sizeof(struct ipv6hdr);
1009	protocol = cpu_to_be16(ETH_P_IPV6);
1010	XFRM_MODE_SKB_CB(skbseq->root_skb)->tos =
1011	ipv6_get_dsfield(ipv6h: (struct ipv6hdr *)iph);
1012	} else if (iph->version == 0x0) {
1013	/* pad */
1014	data = tail;
1015	break;
1016	} else {
1017	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
1018	goto done;
1019	}
1020
1021	if (unlikely(skbseq->stepped_offset)) {
1022	/* We need to reset our seq read, it can't backup at
1023	* this point.
1024	*/
1025	struct sk_buff *save = skbseq->root_skb;
1026
1027	skb_abort_seq_read(st: skbseq);
1028	skb_prepare_seq_read(skb: save, from: data, to: tail, st: skbseq);
1029	}
1030
1031	if (first_skb) {
1032	skb = NULL;
1033	} else {
1034	first_skb = skb;
1035	first_iplen = iplen;
1036	fragwalk = NULL;
1037
1038	/* We are going to skip over `data` bytes to reach the
1039	* start of the IP header of `iphlen` len for `iplen`
1040	* inner packet.
1041	*/
1042
1043	if (skb_has_frag_list(skb)) {
1044	defer = skb;
1045	skb = NULL;
1046	} else if (data + iphlen <= skb_headlen(skb) &&
1047	/* make sure our header is 32-bit aligned? */
1048	/* ((uintptr_t)(skb->data + data) & 0x3) == 0 && */
1049	skb_tailroom(skb) + tail - data >= iplen) {
1050	/* Reuse the received skb.
1051	*
1052	* We have enough headlen to pull past any
1053	* initial fragment data, leaving at least the
1054	* IP header in the linear buffer space.
1055	*
1056	* For linear buffer space we only require that
1057	* linear buffer space is large enough to
1058	* eventually hold the entire reassembled
1059	* packet (by including tailroom in the check).
1060	*
1061	* For non-linear tailroom is 0 and so we only
1062	* re-use if the entire packet is present
1063	* already.
1064	*
1065	* NOTE: there are many more options for
1066	* sharing, KISS for now. Also, this can produce
1067	* skb's with the IP header unaligned to 32
1068	* bits. If that ends up being a problem then a
1069	* check should be added to the conditional
1070	* above that the header lies on a 32-bit
1071	* boundary as well.
1072	*/
1073	skb_pull(skb, len: data);
1074
1075	/* our range just changed */
1076	data = 0;
1077	tail = skb->len;
1078	remaining = skb->len;
1079
1080	skb->protocol = protocol;
1081	skb_mac_header_rebuild(skb);
1082	if (skb->mac_len)
1083	eth_hdr(skb)->h_proto = skb->protocol;
1084
1085	/* all pointers could be changed now reset walk */
1086	skb_abort_seq_read(st: skbseq);
1087	skb_prepare_seq_read(skb, from: data, to: tail, st: skbseq);
1088	} else if (skb->head_frag &&
1089	/* We have the IP header right now */
1090	remaining >= iphlen) {
1091	fragwalk = &_fragwalk;
1092	iptfs_skb_prepare_frag_walk(skb, initial_offset: data, walk: fragwalk);
1093	defer = skb;
1094	skb = NULL;
1095	} else {
1096	/* We couldn't reuse the input skb so allocate a
1097	* new one.
1098	*/
1099	defer = skb;
1100	skb = NULL;
1101	}
1102
1103	/* Don't trim `first_skb` until the end as we are
1104	* walking that data now.
1105	*/
1106	}
1107
1108	capturelen = min(iplen, remaining);
1109	if (!skb) {
1110	if (!fragwalk ||
1111	/* Large enough to be worth sharing */
1112	iplen < IPTFS_PKT_SHARE_MIN ||
1113	/* Have IP header + some data to share. */
1114	capturelen <= iphlen ||
1115	/* Try creating skb and adding frags */
1116	!(skb = iptfs_pskb_add_frags(tpl: first_skb, walk: fragwalk,
1117	off: data, len: capturelen,
1118	st: skbseq, copy_len: iphlen))) {
1119	skb = iptfs_pskb_extract_seq(skblen: iplen, st: skbseq, off: data, len: capturelen);
1120	}
1121	if (!skb) {
1122	/* skip to next packet or done */
1123	data += capturelen;
1124	continue;
1125	}
1126
1127	skb->protocol = protocol;
1128	if (old_mac) {
1129	/* rebuild the mac header */
1130	skb_set_mac_header(skb, offset: -first_skb->mac_len);
1131	memcpy(skb_mac_header(skb), old_mac, first_skb->mac_len);
1132	eth_hdr(skb)->h_proto = skb->protocol;
1133	}
1134	}
1135
1136	data += capturelen;
1137
1138	if (skb->len < iplen) {
1139	/* Start reassembly */
1140	spin_lock(lock: &xtfs->drop_lock);
1141
1142	xtfs->ra_newskb = skb;
1143	xtfs->ra_wantseq = seq + 1;
1144	if (!hrtimer_is_queued(timer: &xtfs->drop_timer)) {
1145	/* softirq blocked lest the timer fire and interrupt us */
1146	hrtimer_start(timer: &xtfs->drop_timer,
1147	tim: xtfs->drop_time_ns,
1148	IPTFS_HRTIMER_MODE);
1149	}
1150
1151	spin_unlock(lock: &xtfs->drop_lock);
1152
1153	break;
1154	}
1155
1156	iptfs_complete_inner_skb(x, skb);
1157	list_add_tail(new: &skb->list, head: sublist);
1158	}
1159
1160	if (data != tail)
1161	/* this should not happen from the above code */
1162	XFRM_INC_STATS(net, LINUX_MIB_XFRMINIPTFSERROR);
1163
1164	if (first_skb && first_iplen && !defer && first_skb != xtfs->ra_newskb) {
1165	/* first_skb is queued b/c !defer and not partial */
1166	if (pskb_trim(skb: first_skb, len: first_iplen)) {
1167	/* error trimming */
1168	list_del(entry: &first_skb->list);
1169	defer = first_skb;
1170	}
1171	first_skb->ip_summed = CHECKSUM_NONE;
1172	}
1173
1174	/* Send the packets! */
1175	list_for_each_entry_safe(skb, next, sublist, list) {
1176	skb_list_del_init(skb);
1177	if (xfrm_input(skb, nexthdr: 0, spi: 0, encap_type: -2))
1178	kfree_skb(skb);
1179	}
1180	done:
1181	skb = skbseq->root_skb;
1182	skb_abort_seq_read(st: skbseq);
1183
1184	if (defer) {
1185	consume_skb(skb: defer);
1186	} else if (!first_skb) {
1187	/* skb is the original passed in skb, but we didn't get far
1188	* enough to process it as the first_skb, if we had it would
1189	* either be save in ra_newskb, trimmed and sent on as an skb or
1190	* placed in defer to be freed.
1191	*/
1192	kfree_skb(skb);
1193	}
1194	return true;
1195	}
1196
1197	/**
1198	* iptfs_input_ordered() - handle next in order IPTFS payload.
1199	* @x: xfrm state
1200	* @skb: current packet
1201	*
1202	* Process the IPTFS payload in `skb` and consume it afterwards.
1203	*/
1204	static void iptfs_input_ordered(struct xfrm_state *x, struct sk_buff *skb)
1205	{
1206	struct ip_iptfs_cc_hdr iptcch;
1207	struct skb_seq_state skbseq;
1208	struct list_head sublist; /* rename this it's just a list */
1209	struct xfrm_iptfs_data *xtfs;
1210	struct ip_iptfs_hdr *ipth;
1211	struct net *net;
1212	u32 blkoff, data, remaining;
1213	bool consumed = false;
1214	u64 seq;
1215
1216	xtfs = x->mode_data;
1217	net = xs_net(x);
1218
1219	seq = __esp_seq(skb);
1220
1221	/* Large enough to hold both types of header */
1222	ipth = (struct ip_iptfs_hdr *)&iptcch;
1223
1224	skb_prepare_seq_read(skb, from: 0, to: skb->len, st: &skbseq);
1225
1226	/* Get the IPTFS header and validate it */
1227
1228	if (skb_copy_seq_read(st: &skbseq, offset: 0, to: ipth, len: sizeof(*ipth))) {
1229	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
1230	goto done;
1231	}
1232	data = sizeof(*ipth);
1233
1234	trace_iptfs_egress_recv(skb, xtfs, be16_to_cpu(ipth->block_offset));
1235
1236	/* Set data past the basic header */
1237	if (ipth->subtype == IPTFS_SUBTYPE_CC) {
1238	/* Copy the rest of the CC header */
1239	remaining = sizeof(iptcch) - sizeof(*ipth);
1240	if (skb_copy_seq_read(st: &skbseq, offset: data, to: ipth + 1, len: remaining)) {
1241	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
1242	goto done;
1243	}
1244	data += remaining;
1245	} else if (ipth->subtype != IPTFS_SUBTYPE_BASIC) {
1246	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
1247	goto done;
1248	}
1249
1250	if (ipth->flags != 0) {
1251	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
1252	goto done;
1253	}
1254
1255	INIT_LIST_HEAD(list: &sublist);
1256
1257	/* Handle fragment at start of payload, and/or waiting reassembly. */
1258
1259	blkoff = ntohs(ipth->block_offset);
1260	/* check before locking i.e., maybe */
1261	if (blkoff || xtfs->ra_runtlen || xtfs->ra_newskb) {
1262	spin_lock(lock: &xtfs->drop_lock);
1263
1264	/* check again after lock */
1265	if (blkoff || xtfs->ra_runtlen || xtfs->ra_newskb) {
1266	data = iptfs_reassem_cont(xtfs, seq, st: &skbseq, skb, data,
1267	blkoff, list: &sublist);
1268	}
1269
1270	spin_unlock(lock: &xtfs->drop_lock);
1271	}
1272
1273	/* New packets */
1274	consumed = __input_process_payload(x, data, skbseq: &skbseq, sublist: &sublist);
1275	done:
1276	if (!consumed) {
1277	skb = skbseq.root_skb;
1278	skb_abort_seq_read(st: &skbseq);
1279	kfree_skb(skb);
1280	}
1281	}
1282
1283	/* ------------------------------- */
1284	/* Input (Egress) Re-ordering Code */
1285	/* ------------------------------- */
1286
1287	static void __vec_shift(struct xfrm_iptfs_data *xtfs, u32 shift)
1288	{
1289	u32 savedlen = xtfs->w_savedlen;
1290
1291	if (shift > savedlen)
1292	shift = savedlen;
1293	if (shift != savedlen)
1294	memcpy(xtfs->w_saved, xtfs->w_saved + shift,
1295	(savedlen - shift) * sizeof(*xtfs->w_saved));
1296	memset(xtfs->w_saved + savedlen - shift, 0,
1297	shift * sizeof(*xtfs->w_saved));
1298	xtfs->w_savedlen -= shift;
1299	}
1300
1301	static void __reorder_past(struct xfrm_iptfs_data *xtfs, struct sk_buff *inskb,
1302	struct list_head *freelist)
1303	{
1304	list_add_tail(new: &inskb->list, head: freelist);
1305	}
1306
1307	static u32 __reorder_drop(struct xfrm_iptfs_data *xtfs, struct list_head *list)
1308
1309	{
1310	struct skb_wseq *s, *se;
1311	const u32 savedlen = xtfs->w_savedlen;
1312	time64_t now = ktime_get_raw_fast_ns();
1313	u32 count = 0;
1314	u32 scount = 0;
1315
1316	if (xtfs->w_saved[0].drop_time > now)
1317	goto set_timer;
1318
1319	++xtfs->w_wantseq;
1320
1321	/* Keep flushing packets until we reach a drop time greater than now. */
1322	s = xtfs->w_saved;
1323	se = s + savedlen;
1324	do {
1325	/* Walking past empty slots until we reach a packet */
1326	for (; s < se && !s->skb; s++) {
1327	if (s->drop_time > now)
1328	goto outerdone;
1329	}
1330	/* Sending packets until we hit another empty slot. */
1331	for (; s < se && s->skb; scount++, s++)
1332	list_add_tail(new: &s->skb->list, head: list);
1333	} while (s < se);
1334	outerdone:
1335
1336	count = s - xtfs->w_saved;
1337	if (count) {
1338	xtfs->w_wantseq += count;
1339
1340	/* Shift handled slots plus final empty slot into slot 0. */
1341	__vec_shift(xtfs, shift: count);
1342	}
1343
1344	if (xtfs->w_savedlen) {
1345	set_timer:
1346	/* Drifting is OK */
1347	hrtimer_start(timer: &xtfs->drop_timer,
1348	tim: xtfs->w_saved[0].drop_time - now,
1349	IPTFS_HRTIMER_MODE);
1350	}
1351	return scount;
1352	}
1353
1354	static void __reorder_this(struct xfrm_iptfs_data *xtfs, struct sk_buff *inskb,
1355	struct list_head *list)
1356	{
1357	struct skb_wseq *s, *se;
1358	const u32 savedlen = xtfs->w_savedlen;
1359	u32 count = 0;
1360
1361	/* Got what we wanted. */
1362	list_add_tail(new: &inskb->list, head: list);
1363	++xtfs->w_wantseq;
1364	if (!savedlen)
1365	return;
1366
1367	/* Flush remaining consecutive packets. */
1368
1369	/* Keep sending until we hit another missed pkt. */
1370	for (s = xtfs->w_saved, se = s + savedlen; s < se && s->skb; s++)
1371	list_add_tail(new: &s->skb->list, head: list);
1372	count = s - xtfs->w_saved;
1373	if (count)
1374	xtfs->w_wantseq += count;
1375
1376	/* Shift handled slots plus final empty slot into slot 0. */
1377	__vec_shift(xtfs, shift: count + 1);
1378	}
1379
1380	/* Set the slot's drop time and all the empty slots below it until reaching a
1381	* filled slot which will already be set.
1382	*/
1383	static void iptfs_set_window_drop_times(struct xfrm_iptfs_data *xtfs, int index)
1384	{
1385	const u32 savedlen = xtfs->w_savedlen;
1386	struct skb_wseq *s = xtfs->w_saved;
1387	time64_t drop_time;
1388
1389	assert_spin_locked(&xtfs->drop_lock);
1390
1391	if (savedlen > index + 1) {
1392	/* we are below another, our drop time and the timer are already set */
1393	return;
1394	}
1395	/* we are the most future so get a new drop time. */
1396	drop_time = ktime_get_raw_fast_ns();
1397	drop_time += xtfs->drop_time_ns;
1398
1399	/* Walk back through the array setting drop times as we go */
1400	s[index].drop_time = drop_time;
1401	while (index-- > 0 && !s[index].skb)
1402	s[index].drop_time = drop_time;
1403
1404	/* If we walked all the way back, schedule the drop timer if needed */
1405	if (index == -1 && !hrtimer_is_queued(timer: &xtfs->drop_timer))
1406	hrtimer_start(timer: &xtfs->drop_timer, tim: xtfs->drop_time_ns,
1407	IPTFS_HRTIMER_MODE);
1408	}
1409
1410	static void __reorder_future_fits(struct xfrm_iptfs_data *xtfs,
1411	struct sk_buff *inskb,
1412	struct list_head *freelist)
1413	{
1414	const u64 inseq = __esp_seq(skb: inskb);
1415	const u64 wantseq = xtfs->w_wantseq;
1416	const u64 distance = inseq - wantseq;
1417	const u32 savedlen = xtfs->w_savedlen;
1418	const u32 index = distance - 1;
1419
1420	/* Handle future sequence number received which fits in the window.
1421	*
1422	* We know we don't have the seq we want so we won't be able to flush
1423	* anything.
1424	*/
1425
1426	/* slot count is 4, saved size is 3 savedlen is 2
1427	*
1428	* "window boundary" is based on the fixed window size
1429	* distance is also slot number
1430	* index is an array index (i.e., - 1 of slot)
1431	* : : - implicit NULL after array len
1432	*
1433	* +--------- used length (savedlen == 2)
1434	* | +----- array size (nslots - 1 == 3)
1435	* | | + window boundary (nslots == 4)
1436	* V V | V
1437	* |
1438	* 0 1 2 3 | slot number
1439	* --- 0 1 2 | array index
1440	* [-] [b] : :| array
1441	*
1442	* "2" "3" "4" *5*| seq numbers
1443	*
1444	* We receive seq number 5
1445	* distance == 3 [inseq(5) - w_wantseq(2)]
1446	* index == 2 [distance(6) - 1]
1447	*/
1448
1449	if (xtfs->w_saved[index].skb) {
1450	/* a dup of a future */
1451	list_add_tail(new: &inskb->list, head: freelist);
1452	return;
1453	}
1454
1455	xtfs->w_saved[index].skb = inskb;
1456	xtfs->w_savedlen = max(savedlen, index + 1);
1457	iptfs_set_window_drop_times(xtfs, index);
1458	}
1459
1460	static void __reorder_future_shifts(struct xfrm_iptfs_data *xtfs,
1461	struct sk_buff *inskb,
1462	struct list_head *list)
1463	{
1464	const u32 nslots = xtfs->cfg.reorder_win_size + 1;
1465	const u64 inseq = __esp_seq(skb: inskb);
1466	u32 savedlen = xtfs->w_savedlen;
1467	u64 wantseq = xtfs->w_wantseq;
1468	struct skb_wseq *wnext;
1469	struct sk_buff *slot0;
1470	u32 beyond, shifting, slot;
1471	u64 distance;
1472
1473	/* Handle future sequence number received.
1474	*
1475	* IMPORTANT: we are at least advancing w_wantseq (i.e., wantseq) by 1
1476	* b/c we are beyond the window boundary.
1477	*
1478	* We know we don't have the wantseq so that counts as a drop.
1479	*/
1480
1481	/* example: slot count is 4, array size is 3 savedlen is 2, slot 0 is
1482	* the missing sequence number.
1483	*
1484	* the final slot at savedlen (index savedlen - 1) is always occupied.
1485	*
1486	* beyond is "beyond array size" not savedlen.
1487	*
1488	* +--------- array length (savedlen == 2)
1489	* | +----- array size (nslots - 1 == 3)
1490	* | | +- window boundary (nslots == 4)
1491	* V V |
1492	* |
1493	* 0 1 2 3 | slot number
1494	* --- 0 1 2 | array index
1495	* [b] [c] : :| array
1496	* |
1497	* "2" "3" "4" "5"|*6* seq numbers
1498	*
1499	* We receive seq number 6
1500	* distance == 4 [inseq(6) - w_wantseq(2)]
1501	* newslot == distance
1502	* index == 3 [distance(4) - 1]
1503	* beyond == 1 [newslot(4) - lastslot((nslots(4) - 1))]
1504	* shifting == 1 [min(savedlen(2), beyond(1)]
1505	* slot0_skb == [b], and should match w_wantseq
1506	*
1507	* +--- window boundary (nslots == 4)
1508	* 0 1 2 3 | 4 slot number
1509	* --- 0 1 2 | 3 array index
1510	* [b] : : : :| array
1511	* "2" "3" "4" "5" *6* seq numbers
1512	*
1513	* We receive seq number 6
1514	* distance == 4 [inseq(6) - w_wantseq(2)]
1515	* newslot == distance
1516	* index == 3 [distance(4) - 1]
1517	* beyond == 1 [newslot(4) - lastslot((nslots(4) - 1))]
1518	* shifting == 1 [min(savedlen(1), beyond(1)]
1519	* slot0_skb == [b] and should match w_wantseq
1520	*
1521	* +-- window boundary (nslots == 4)
1522	* 0 1 2 3 | 4 5 6 slot number
1523	* --- 0 1 2 | 3 4 5 array index
1524	* [-] [c] : :| array
1525	* "2" "3" "4" "5" "6" "7" *8* seq numbers
1526	*
1527	* savedlen = 2, beyond = 3
1528	* iter 1: slot0 == NULL, missed++, lastdrop = 2 (2+1-1), slot0 = [-]
1529	* iter 2: slot0 == NULL, missed++, lastdrop = 3 (2+2-1), slot0 = [c]
1530	* 2 < 3, extra = 1 (3-2), missed += extra, lastdrop = 4 (2+2+1-1)
1531	*
1532	* We receive seq number 8
1533	* distance == 6 [inseq(8) - w_wantseq(2)]
1534	* newslot == distance
1535	* index == 5 [distance(6) - 1]
1536	* beyond == 3 [newslot(6) - lastslot((nslots(4) - 1))]
1537	* shifting == 2 [min(savedlen(2), beyond(3)]
1538	*
1539	* slot0_skb == NULL changed from [b] when "savedlen < beyond" is true.
1540	*/
1541
1542	/* Now send any packets that are being shifted out of saved, and account
1543	* for missing packets that are exiting the window as we shift it.
1544	*/
1545
1546	distance = inseq - wantseq;
1547	beyond = distance - (nslots - 1);
1548
1549	/* If savedlen > beyond we are shifting some, else all. */
1550	shifting = min(savedlen, beyond);
1551
1552	/* slot0 is the buf that just shifted out and into slot0 */
1553	slot0 = NULL;
1554	wnext = xtfs->w_saved;
1555	for (slot = 1; slot <= shifting; slot++, wnext++) {
1556	/* handle what was in slot0 before we occupy it */
1557	if (slot0)
1558	list_add_tail(new: &slot0->list, head: list);
1559	slot0 = wnext->skb;
1560	wnext->skb = NULL;
1561	}
1562
1563	/* slot0 is now either NULL (in which case it's what we now are waiting
1564	* for, or a buf in which case we need to handle it like we received it;
1565	* however, we may be advancing past that buffer as well..
1566	*/
1567
1568	/* Handle case where we need to shift more than we had saved, slot0 will
1569	* be NULL iff savedlen is 0, otherwise slot0 will always be
1570	* non-NULL b/c we shifted the final element, which is always set if
1571	* there is any saved, into slot0.
1572	*/
1573	if (savedlen < beyond) {
1574	if (savedlen != 0)
1575	list_add_tail(new: &slot0->list, head: list);
1576	slot0 = NULL;
1577	/* slot0 has had an empty slot pushed into it */
1578	}
1579
1580	/* Remove the entries */
1581	__vec_shift(xtfs, shift: beyond);
1582
1583	/* Advance want seq */
1584	xtfs->w_wantseq += beyond;
1585
1586	/* Process drops here when implementing congestion control */
1587
1588	/* We've shifted. plug the packet in at the end. */
1589	xtfs->w_savedlen = nslots - 1;
1590	xtfs->w_saved[xtfs->w_savedlen - 1].skb = inskb;
1591	iptfs_set_window_drop_times(xtfs, index: xtfs->w_savedlen - 1);
1592
1593	/* if we don't have a slot0 then we must wait for it */
1594	if (!slot0)
1595	return;
1596
1597	/* If slot0, seq must match new want seq */
1598
1599	/* slot0 is valid, treat like we received expected. */
1600	__reorder_this(xtfs, inskb: slot0, list);
1601	}
1602
1603	/* Receive a new packet into the reorder window. Return a list of ordered
1604	* packets from the window.
1605	*/
1606	static void iptfs_input_reorder(struct xfrm_iptfs_data *xtfs,
1607	struct sk_buff *inskb, struct list_head *list,
1608	struct list_head *freelist)
1609	{
1610	const u32 nslots = xtfs->cfg.reorder_win_size + 1;
1611	u64 inseq = __esp_seq(skb: inskb);
1612	u64 wantseq;
1613
1614	assert_spin_locked(&xtfs->drop_lock);
1615
1616	if (unlikely(!xtfs->w_seq_set)) {
1617	xtfs->w_seq_set = true;
1618	xtfs->w_wantseq = inseq;
1619	}
1620	wantseq = xtfs->w_wantseq;
1621
1622	if (likely(inseq == wantseq))
1623	__reorder_this(xtfs, inskb, list);
1624	else if (inseq < wantseq)
1625	__reorder_past(xtfs, inskb, freelist);
1626	else if ((inseq - wantseq) < nslots)
1627	__reorder_future_fits(xtfs, inskb, freelist);
1628	else
1629	__reorder_future_shifts(xtfs, inskb, list);
1630	}
1631
1632	/**
1633	* iptfs_drop_timer() - Handle drop timer expiry.
1634	* @me: the timer
1635	*
1636	* This is similar to our input function.
1637	*
1638	* The drop timer is set when we start an in progress reassembly, and also when
1639	* we save a future packet in the window saved array.
1640	*
1641	* NOTE packets in the save window are always newer WRT drop times as
1642	* they get further in the future. i.e. for:
1643	*
1644	* if slots (S0, S1, ... Sn) and `Dn` is the drop time for slot `Sn`,
1645	* then D(n-1) <= D(n).
1646	*
1647	* So, regardless of why the timer is firing we can always discard any inprogress
1648	* fragment; either it's the reassembly timer, or slot 0 is going to be
1649	* dropped as S0 must have the most recent drop time, and slot 0 holds the
1650	* continuation fragment of the in progress packet.
1651	*
1652	* Returns HRTIMER_NORESTART.
1653	*/
1654	static enum hrtimer_restart iptfs_drop_timer(struct hrtimer *me)
1655	{
1656	struct sk_buff *skb, *next;
1657	struct list_head list;
1658	struct xfrm_iptfs_data *xtfs;
1659	struct xfrm_state *x;
1660	u32 count;
1661
1662	xtfs = container_of(me, typeof(*xtfs), drop_timer);
1663	x = xtfs->x;
1664
1665	INIT_LIST_HEAD(list: &list);
1666
1667	spin_lock(lock: &xtfs->drop_lock);
1668
1669	/* Drop any in progress packet */
1670	skb = xtfs->ra_newskb;
1671	xtfs->ra_newskb = NULL;
1672
1673	/* Now drop as many packets as we should from the reordering window
1674	* saved array
1675	*/
1676	count = xtfs->w_savedlen ? __reorder_drop(xtfs, list: &list) : 0;
1677
1678	spin_unlock(lock: &xtfs->drop_lock);
1679
1680	if (skb)
1681	kfree_skb_reason(skb, reason: SKB_DROP_REASON_FRAG_REASM_TIMEOUT);
1682
1683	if (count) {
1684	list_for_each_entry_safe(skb, next, &list, list) {
1685	skb_list_del_init(skb);
1686	iptfs_input_ordered(x, skb);
1687	}
1688	}
1689
1690	return HRTIMER_NORESTART;
1691	}
1692
1693	/**
1694	* iptfs_input() - handle receipt of iptfs payload
1695	* @x: xfrm state
1696	* @skb: the packet
1697	*
1698	* We have an IPTFS payload order it if needed, then process newly in order
1699	* packets.
1700	*
1701	* Return: -EINPROGRESS to inform xfrm_input to stop processing the skb.
1702	*/
1703	static int iptfs_input(struct xfrm_state *x, struct sk_buff *skb)
1704	{
1705	struct list_head freelist, list;
1706	struct xfrm_iptfs_data *xtfs = x->mode_data;
1707	struct sk_buff *next;
1708
1709	/* Fast path for no reorder window. */
1710	if (xtfs->cfg.reorder_win_size == 0) {
1711	iptfs_input_ordered(x, skb);
1712	goto done;
1713	}
1714
1715	/* Fetch list of in-order packets from the reordering window as well as
1716	* a list of buffers we need to now free.
1717	*/
1718	INIT_LIST_HEAD(list: &list);
1719	INIT_LIST_HEAD(list: &freelist);
1720
1721	spin_lock(lock: &xtfs->drop_lock);
1722	iptfs_input_reorder(xtfs, inskb: skb, list: &list, freelist: &freelist);
1723	spin_unlock(lock: &xtfs->drop_lock);
1724
1725	list_for_each_entry_safe(skb, next, &list, list) {
1726	skb_list_del_init(skb);
1727	iptfs_input_ordered(x, skb);
1728	}
1729
1730	list_for_each_entry_safe(skb, next, &freelist, list) {
1731	skb_list_del_init(skb);
1732	kfree_skb(skb);
1733	}
1734	done:
1735	/* We always have dealt with the input SKB, either we are re-using it,
1736	* or we have freed it. Return EINPROGRESS so that xfrm_input stops
1737	* processing it.
1738	*/
1739	return -EINPROGRESS;
1740	}
1741
1742	/* ================================= */
1743	/* IPTFS Sending (ingress) Functions */
1744	/* ================================= */
1745
1746	/* ------------------------- */
1747	/* Enqueue to send functions */
1748	/* ------------------------- */
1749
1750	/**
1751	* iptfs_enqueue() - enqueue packet if ok to send.
1752	* @xtfs: xtfs state
1753	* @skb: the packet
1754	*
1755	* Return: true if packet enqueued.
1756	*/
1757	static bool iptfs_enqueue(struct xfrm_iptfs_data *xtfs, struct sk_buff *skb)
1758	{
1759	u64 newsz = xtfs->queue_size + skb->len;
1760	struct iphdr *iph;
1761
1762	assert_spin_locked(&xtfs->x->lock);
1763
1764	if (newsz > xtfs->cfg.max_queue_size)
1765	return false;
1766
1767	/* Set ECN CE if we are above our ECN queue threshold */
1768	if (newsz > xtfs->ecn_queue_size) {
1769	iph = ip_hdr(skb);
1770	if (iph->version == 4)
1771	IP_ECN_set_ce(iph);
1772	else if (iph->version == 6)
1773	IP6_ECN_set_ce(skb, iph: ipv6_hdr(skb));
1774	}
1775
1776	__skb_queue_tail(list: &xtfs->queue, newsk: skb);
1777	xtfs->queue_size += skb->len;
1778	return true;
1779	}
1780
1781	static int iptfs_get_cur_pmtu(struct xfrm_state *x, struct xfrm_iptfs_data *xtfs,
1782	struct sk_buff *skb)
1783	{
1784	struct xfrm_dst *xdst = (struct xfrm_dst *)skb_dst(skb);
1785	u32 payload_mtu = xtfs->payload_mtu;
1786	u32 pmtu = __iptfs_get_inner_mtu(x, outer_mtu: xdst->child_mtu_cached);
1787
1788	if (payload_mtu && payload_mtu < pmtu)
1789	pmtu = payload_mtu;
1790
1791	return pmtu;
1792	}
1793
1794	static int iptfs_is_too_big(struct sock *sk, struct sk_buff *skb, u32 pmtu)
1795	{
1796	if (skb->len <= pmtu)
1797	return 0;
1798
1799	/* We only send ICMP too big if the user has configured us as
1800	* dont-fragment.
1801	*/
1802	if (skb->dev)
1803	XFRM_INC_STATS(dev_net(skb->dev), LINUX_MIB_XFRMOUTERROR);
1804
1805	if (sk)
1806	xfrm_local_error(skb, mtu: pmtu);
1807	else if (ip_hdr(skb)->version == 4)
1808	icmp_send(skb_in: skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(pmtu));
1809	else
1810	icmpv6_send(skb, ICMPV6_PKT_TOOBIG, code: 0, info: pmtu);
1811
1812	return 1;
1813	}
1814
1815	/* IPv4/IPv6 packet ingress to IPTFS tunnel, arrange to send in IPTFS payload
1816	* (i.e., aggregating or fragmenting as appropriate).
1817	* This is set in dst->output for an SA.
1818	*/
1819	static int iptfs_output_collect(struct net *net, struct sock *sk, struct sk_buff *skb)
1820	{
1821	struct dst_entry *dst = skb_dst(skb);
1822	struct xfrm_state *x = dst->xfrm;
1823	struct xfrm_iptfs_data *xtfs = x->mode_data;
1824	struct sk_buff *segs, *nskb;
1825	u32 pmtu = 0;
1826	bool ok = true;
1827	bool was_gso;
1828
1829	/* We have hooked into dst_entry->output which means we have skipped the
1830	* protocol specific netfilter (see xfrm4_output, xfrm6_output).
1831	* when our timer runs we will end up calling xfrm_output directly on
1832	* the encapsulated traffic.
1833	*
1834	* For both cases this is the NF_INET_POST_ROUTING hook which allows
1835	* changing the skb->dst entry which then may not be xfrm based anymore
1836	* in which case a REROUTED flag is set. and dst_output is called.
1837	*
1838	* For IPv6 we are also skipping fragmentation handling for local
1839	* sockets, which may or may not be good depending on our tunnel DF
1840	* setting. Normally with fragmentation supported we want to skip this
1841	* fragmentation.
1842	*/
1843
1844	if (xtfs->cfg.dont_frag)
1845	pmtu = iptfs_get_cur_pmtu(x, xtfs, skb);
1846
1847	/* Break apart GSO skbs. If the queue is nearing full then we want the
1848	* accounting and queuing to be based on the individual packets not on the
1849	* aggregate GSO buffer.
1850	*/
1851	was_gso = skb_is_gso(skb);
1852	if (!was_gso) {
1853	segs = skb;
1854	} else {
1855	segs = skb_gso_segment(skb, features: 0);
1856	if (IS_ERR_OR_NULL(ptr: segs)) {
1857	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
1858	kfree_skb(skb);
1859	if (IS_ERR(ptr: segs))
1860	return PTR_ERR(ptr: segs);
1861	return -EINVAL;
1862	}
1863	consume_skb(skb);
1864	skb = NULL;
1865	}
1866
1867	/* We can be running on multiple cores and from the network softirq or
1868	* from user context depending on where the packet is coming from.
1869	*/
1870	spin_lock_bh(lock: &x->lock);
1871
1872	skb_list_walk_safe(segs, skb, nskb) {
1873	skb_mark_not_on_list(skb);
1874
1875	/* Once we drop due to no queue space we continue to drop the
1876	* rest of the packets from that GRO.
1877	*/
1878	if (!ok) {
1879	nospace:
1880	trace_iptfs_no_queue_space(skb, xtfs, pmtu, was_gso);
1881	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOQSPACE);
1882	kfree_skb_reason(skb, reason: SKB_DROP_REASON_FULL_RING);
1883	continue;
1884	}
1885
1886	/* If the user indicated no iptfs fragmenting check before
1887	* enqueue.
1888	*/
1889	if (xtfs->cfg.dont_frag && iptfs_is_too_big(sk, skb, pmtu)) {
1890	trace_iptfs_too_big(skb, xtfs, pmtu, was_gso);
1891	kfree_skb_reason(skb, reason: SKB_DROP_REASON_PKT_TOO_BIG);
1892	continue;
1893	}
1894
1895	/* Enqueue to send in tunnel */
1896	ok = iptfs_enqueue(xtfs, skb);
1897	if (!ok)
1898	goto nospace;
1899
1900	trace_iptfs_enqueue(skb, xtfs, pmtu, was_gso);
1901	}
1902
1903	/* Start a delay timer if we don't have one yet */
1904	if (!hrtimer_is_queued(timer: &xtfs->iptfs_timer)) {
1905	hrtimer_start(timer: &xtfs->iptfs_timer, tim: xtfs->init_delay_ns, IPTFS_HRTIMER_MODE);
1906	xtfs->iptfs_settime = ktime_get_raw_fast_ns();
1907	trace_iptfs_timer_start(xtfs, time_val: xtfs->init_delay_ns);
1908	}
1909
1910	spin_unlock_bh(lock: &x->lock);
1911	return 0;
1912	}
1913
1914	/* -------------------------- */
1915	/* Dequeue and send functions */
1916	/* -------------------------- */
1917
1918	static void iptfs_output_prepare_skb(struct sk_buff *skb, u32 blkoff)
1919	{
1920	struct ip_iptfs_hdr *h;
1921	size_t hsz = sizeof(*h);
1922
1923	/* now reset values to be pointing at the rest of the packets */
1924	h = skb_push(skb, len: hsz);
1925	memset(h, 0, hsz);
1926	if (blkoff)
1927	h->block_offset = htons(blkoff);
1928
1929	/* network_header current points at the inner IP packet
1930	* move it to the iptfs header
1931	*/
1932	skb->transport_header = skb->network_header;
1933	skb->network_header -= hsz;
1934
1935	IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE;
1936	}
1937
1938	/**
1939	* iptfs_copy_create_frag() - create an inner fragment skb.
1940	* @st: The source packet data.
1941	* @offset: offset in @st of the new fragment data.
1942	* @copy_len: the amount of data to copy from @st.
1943	*
1944	* Create a new skb holding a single IPTFS inner packet fragment. @copy_len must
1945	* not be greater than the max fragment size.
1946	*
1947	* Return: the new fragment skb or an ERR_PTR().
1948	*/
1949	static struct sk_buff *iptfs_copy_create_frag(struct skb_seq_state *st, u32 offset, u32 copy_len)
1950	{
1951	struct sk_buff *src = st->root_skb;
1952	struct sk_buff *skb;
1953	int err;
1954
1955	skb = iptfs_alloc_skb(tpl: src, len: copy_len, l3resv: true);
1956	if (!skb)
1957	return ERR_PTR(error: -ENOMEM);
1958
1959	/* Now copy `copy_len` data from src */
1960	err = skb_copy_seq_read(st, offset, to: skb_put(skb, len: copy_len), len: copy_len);
1961	if (err) {
1962	kfree_skb(skb);
1963	return ERR_PTR(error: err);
1964	}
1965
1966	return skb;
1967	}
1968
1969	/**
1970	* iptfs_copy_create_frags() - create and send N-1 fragments of a larger skb.
1971	* @skbp: the source packet skb (IN), skb holding the last fragment in
1972	* the fragment stream (OUT).
1973	* @xtfs: IPTFS SA state.
1974	* @mtu: the max IPTFS fragment size.
1975	*
1976	* This function is responsible for fragmenting a larger inner packet into a
1977	* sequence of IPTFS payload packets. The last fragment is returned rather than
1978	* being sent so that the caller can append more inner packets (aggregation) if
1979	* there is room.
1980	*
1981	* Return: 0 on success or a negative error code on failure
1982	*/
1983	static int iptfs_copy_create_frags(struct sk_buff **skbp, struct xfrm_iptfs_data *xtfs, u32 mtu)
1984	{
1985	struct skb_seq_state skbseq;
1986	struct list_head sublist;
1987	struct sk_buff *skb = *skbp;
1988	struct sk_buff *nskb = *skbp;
1989	u32 copy_len, offset;
1990	u32 to_copy = skb->len - mtu;
1991	u32 blkoff = 0;
1992	int err = 0;
1993
1994	INIT_LIST_HEAD(list: &sublist);
1995
1996	skb_prepare_seq_read(skb, from: 0, to: skb->len, st: &skbseq);
1997
1998	/* A trimmed `skb` will be sent as the first fragment, later. */
1999	offset = mtu;
2000	to_copy = skb->len - offset;
2001	while (to_copy) {
2002	/* Send all but last fragment to allow agg. append */
2003	trace_iptfs_first_fragmenting(skb: nskb, mtu, blkoff: to_copy, NULL);
2004	list_add_tail(new: &nskb->list, head: &sublist);
2005
2006	/* FUTURE: if the packet has an odd/non-aligning length we could
2007	* send less data in the penultimate fragment so that the last
2008	* fragment then ends on an aligned boundary.
2009	*/
2010	copy_len = min(to_copy, mtu);
2011	nskb = iptfs_copy_create_frag(st: &skbseq, offset, copy_len);
2012	if (IS_ERR(ptr: nskb)) {
2013	XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMOUTERROR);
2014	skb_abort_seq_read(st: &skbseq);
2015	err = PTR_ERR(ptr: nskb);
2016	nskb = NULL;
2017	break;
2018	}
2019	iptfs_output_prepare_skb(skb: nskb, blkoff: to_copy);
2020	offset += copy_len;
2021	to_copy -= copy_len;
2022	blkoff = to_copy;
2023	}
2024	skb_abort_seq_read(st: &skbseq);
2025
2026	/* return last fragment that will be unsent (or NULL) */
2027	*skbp = nskb;
2028	if (nskb)
2029	trace_iptfs_first_final_fragment(skb: nskb, mtu, blkoff, NULL);
2030
2031	/* trim the original skb to MTU */
2032	if (!err)
2033	err = pskb_trim(skb, len: mtu);
2034
2035	if (err) {
2036	/* Free all frags. Don't bother sending a partial packet we will
2037	* never complete.
2038	*/
2039	kfree_skb(skb: nskb);
2040	list_for_each_entry_safe(skb, nskb, &sublist, list) {
2041	skb_list_del_init(skb);
2042	kfree_skb(skb);
2043	}
2044	return err;
2045	}
2046
2047	/* prepare the initial fragment with an iptfs header */
2048	iptfs_output_prepare_skb(skb, blkoff: 0);
2049
2050	/* Send all but last fragment, if we fail to send a fragment then free
2051	* the rest -- no point in sending a packet that can't be reassembled.
2052	*/
2053	list_for_each_entry_safe(skb, nskb, &sublist, list) {
2054	skb_list_del_init(skb);
2055	if (!err)
2056	err = xfrm_output(NULL, skb);
2057	else
2058	kfree_skb(skb);
2059	}
2060	if (err)
2061	kfree_skb(skb: *skbp);
2062	return err;
2063	}
2064
2065	/**
2066	* iptfs_first_skb() - handle the first dequeued inner packet for output
2067	* @skbp: the source packet skb (IN), skb holding the last fragment in
2068	* the fragment stream (OUT).
2069	* @xtfs: IPTFS SA state.
2070	* @mtu: the max IPTFS fragment size.
2071	*
2072	* This function is responsible for fragmenting a larger inner packet into a
2073	* sequence of IPTFS payload packets.
2074	*
2075	* The last fragment is returned rather than being sent so that the caller can
2076	* append more inner packets (aggregation) if there is room.
2077	*
2078	* Return: 0 on success or a negative error code on failure
2079	*/
2080	static int iptfs_first_skb(struct sk_buff **skbp, struct xfrm_iptfs_data *xtfs, u32 mtu)
2081	{
2082	struct sk_buff *skb = *skbp;
2083	int err;
2084
2085	/* Classic ESP skips the don't fragment ICMP error if DF is clear on
2086	* the inner packet or ignore_df is set. Otherwise it will send an ICMP
2087	* or local error if the inner packet won't fit it's MTU.
2088	*
2089	* With IPTFS we do not care about the inner packet DF bit. If the
2090	* tunnel is configured to "don't fragment" we error back if things
2091	* don't fit in our max packet size. Otherwise we iptfs-fragment as
2092	* normal.
2093	*/
2094
2095	/* The opportunity for HW offload has ended */
2096	if (skb->ip_summed == CHECKSUM_PARTIAL) {
2097	err = skb_checksum_help(skb);
2098	if (err)
2099	return err;
2100	}
2101
2102	/* We've split gso up before queuing */
2103
2104	trace_iptfs_first_dequeue(skb, mtu, blkoff: 0, iph: ip_hdr(skb));
2105
2106	/* Consider the buffer Tx'd and no longer owned */
2107	skb_orphan(skb);
2108
2109	/* Simple case -- it fits. `mtu` accounted for all the overhead
2110	* including the basic IPTFS header.
2111	*/
2112	if (skb->len <= mtu) {
2113	iptfs_output_prepare_skb(skb, blkoff: 0);
2114	return 0;
2115	}
2116
2117	return iptfs_copy_create_frags(skbp, xtfs, mtu);
2118	}
2119
2120	static struct sk_buff **iptfs_rehome_fraglist(struct sk_buff **nextp, struct sk_buff *child)
2121	{
2122	u32 fllen = 0;
2123
2124	/* It might be possible to account for a frag list in addition to page
2125	* fragment if it's a valid state to be in. The page fragments size
2126	* should be kept as data_len so only the frag_list size is removed,
2127	* this must be done above as well.
2128	*/
2129	*nextp = skb_shinfo(child)->frag_list;
2130	while (*nextp) {
2131	fllen += (*nextp)->len;
2132	nextp = &(*nextp)->next;
2133	}
2134	skb_frag_list_init(skb: child);
2135	child->len -= fllen;
2136	child->data_len -= fllen;
2137
2138	return nextp;
2139	}
2140
2141	static void iptfs_consume_frags(struct sk_buff *to, struct sk_buff *from)
2142	{
2143	struct skb_shared_info *fromi = skb_shinfo(from);
2144	struct skb_shared_info *toi = skb_shinfo(to);
2145	unsigned int new_truesize;
2146
2147	/* If we have data in a head page, grab it */
2148	if (!skb_headlen(skb: from)) {
2149	new_truesize = SKB_TRUESIZE(skb_end_offset(from));
2150	} else {
2151	iptfs_skb_head_to_frag(skb: from, frag: &toi->frags[toi->nr_frags]);
2152	skb_frag_ref(skb: to, f: toi->nr_frags++);
2153	new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff));
2154	}
2155
2156	/* Move any other page fragments rather than copy */
2157	memcpy(&toi->frags[toi->nr_frags], fromi->frags,
2158	sizeof(fromi->frags[0]) * fromi->nr_frags);
2159	toi->nr_frags += fromi->nr_frags;
2160	fromi->nr_frags = 0;
2161	from->data_len = 0;
2162	from->len = 0;
2163	to->truesize += from->truesize - new_truesize;
2164	from->truesize = new_truesize;
2165
2166	/* We are done with this SKB */
2167	consume_skb(skb: from);
2168	}
2169
2170	static void iptfs_output_queued(struct xfrm_state *x, struct sk_buff_head *list)
2171	{
2172	struct xfrm_iptfs_data *xtfs = x->mode_data;
2173	struct sk_buff *skb, *skb2, **nextp;
2174	struct skb_shared_info *shi, *shi2;
2175
2176	/* If we are fragmenting due to a large inner packet we will output all
2177	* the outer IPTFS packets required to contain the fragments of the
2178	* single large inner packet. These outer packets need to be sent
2179	* consecutively (ESP seq-wise). Since this output function is always
2180	* running from a timer we do not need a lock to provide this guarantee.
2181	* We will output our packets consecutively before the timer is allowed
2182	* to run again on some other CPU.
2183	*/
2184
2185	while ((skb = __skb_dequeue(list))) {
2186	u32 mtu = iptfs_get_cur_pmtu(x, xtfs, skb);
2187	bool share_ok = true;
2188	int remaining;
2189
2190	/* protocol comes to us cleared sometimes */
2191	skb->protocol = x->outer_mode.family == AF_INET ? htons(ETH_P_IP) :
2192	htons(ETH_P_IPV6);
2193
2194	if (skb->len > mtu && xtfs->cfg.dont_frag) {
2195	/* We handle this case before enqueueing so we are only
2196	* here b/c MTU changed after we enqueued before we
2197	* dequeued, just drop these.
2198	*/
2199	XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTERROR);
2200
2201	trace_iptfs_first_toobig(skb, mtu, blkoff: 0, iph: ip_hdr(skb));
2202	kfree_skb_reason(skb, reason: SKB_DROP_REASON_PKT_TOO_BIG);
2203	continue;
2204	}
2205
2206	/* Convert first inner packet into an outer IPTFS packet,
2207	* dealing with any fragmentation into multiple outer packets
2208	* if necessary.
2209	*/
2210	if (iptfs_first_skb(skbp: &skb, xtfs, mtu))
2211	continue;
2212
2213	/* If fragmentation was required the returned skb is the last
2214	* IPTFS fragment in the chain, and it's IPTFS header blkoff has
2215	* been set just past the end of the fragment data.
2216	*
2217	* In either case the space remaining to send more inner packet
2218	* data is `mtu` - (skb->len - sizeof iptfs header). This is b/c
2219	* the `mtu` value has the basic IPTFS header len accounted for,
2220	* and we added that header to the skb so it is a part of
2221	* skb->len, thus we subtract it from the skb length.
2222	*/
2223	remaining = mtu - (skb->len - sizeof(struct ip_iptfs_hdr));
2224
2225	/* Re-home (un-nest) nested fragment lists. We need to do this
2226	* b/c we will simply be appending any following aggregated
2227	* inner packets using the frag list.
2228	*/
2229	shi = skb_shinfo(skb);
2230	nextp = &shi->frag_list;
2231	while (*nextp) {
2232	if (skb_has_frag_list(skb: *nextp))
2233	nextp = iptfs_rehome_fraglist(nextp: &(*nextp)->next, child: *nextp);
2234	else
2235	nextp = &(*nextp)->next;
2236	}
2237
2238	if (shi->frag_list || skb_cloned(skb) || skb_shared(skb))
2239	share_ok = false;
2240
2241	/* See if we have enough space to simply append.
2242	*
2243	* NOTE: Maybe do not append if we will be mis-aligned,
2244	* SW-based endpoints will probably have to copy in this
2245	* case.
2246	*/
2247	while ((skb2 = skb_peek(list_: list))) {
2248	trace_iptfs_ingress_nth_peek(skb: skb2, remaining);
2249	if (skb2->len > remaining)
2250	break;
2251
2252	__skb_unlink(skb: skb2, list);
2253
2254	/* Consider the buffer Tx'd and no longer owned */
2255	skb_orphan(skb);
2256
2257	/* If we don't have a cksum in the packet we need to add
2258	* one before encapsulation.
2259	*/
2260	if (skb2->ip_summed == CHECKSUM_PARTIAL) {
2261	if (skb_checksum_help(skb: skb2)) {
2262	XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTERROR);
2263	kfree_skb(skb: skb2);
2264	continue;
2265	}
2266	}
2267
2268	/* skb->pp_recycle is passed to __skb_flag_unref for all
2269	* frag pages so we can only share pages with skb's who
2270	* match ourselves.
2271	*/
2272	shi2 = skb_shinfo(skb2);
2273	if (share_ok &&
2274	(shi2->frag_list ||
2275	(!skb2->head_frag && skb_headlen(skb)) ||
2276	skb->pp_recycle != skb2->pp_recycle ||
2277	skb_zcopy(skb: skb2) ||
2278	(shi->nr_frags + shi2->nr_frags + 1 > MAX_SKB_FRAGS)))
2279	share_ok = false;
2280
2281	/* Do accounting */
2282	skb->data_len += skb2->len;
2283	skb->len += skb2->len;
2284	remaining -= skb2->len;
2285
2286	trace_iptfs_ingress_nth_add(skb: skb2, share_ok);
2287
2288	if (share_ok) {
2289	iptfs_consume_frags(to: skb, from: skb2);
2290	} else {
2291	/* Append to the frag_list */
2292	*nextp = skb2;
2293	nextp = &skb2->next;
2294	if (skb_has_frag_list(skb: skb2))
2295	nextp = iptfs_rehome_fraglist(nextp,
2296	child: skb2);
2297	skb->truesize += skb2->truesize;
2298	}
2299	}
2300
2301	xfrm_output(NULL, skb);
2302	}
2303	}
2304
2305	static enum hrtimer_restart iptfs_delay_timer(struct hrtimer *me)
2306	{
2307	struct sk_buff_head list;
2308	struct xfrm_iptfs_data *xtfs;
2309	struct xfrm_state *x;
2310	time64_t settime;
2311
2312	xtfs = container_of(me, typeof(*xtfs), iptfs_timer);
2313	x = xtfs->x;
2314
2315	/* Process all the queued packets
2316	*
2317	* softirq execution order: timer > tasklet > hrtimer
2318	*
2319	* Network rx will have run before us giving one last chance to queue
2320	* ingress packets for us to process and transmit.
2321	*/
2322
2323	spin_lock(lock: &x->lock);
2324	__skb_queue_head_init(list: &list);
2325	skb_queue_splice_init(list: &xtfs->queue, head: &list);
2326	xtfs->queue_size = 0;
2327	settime = xtfs->iptfs_settime;
2328	spin_unlock(lock: &x->lock);
2329
2330	/* After the above unlock, packets can begin queuing again, and the
2331	* timer can be set again, from another CPU either in softirq or user
2332	* context (not from this one since we are running at softirq level
2333	* already).
2334	*/
2335
2336	trace_iptfs_timer_expire(xtfs, time_val: (unsigned long long)(ktime_get_raw_fast_ns() - settime));
2337
2338	iptfs_output_queued(x, list: &list);
2339
2340	return HRTIMER_NORESTART;
2341	}
2342
2343	/**
2344	* iptfs_encap_add_ipv4() - add outer encaps
2345	* @x: xfrm state
2346	* @skb: the packet
2347	*
2348	* This was originally taken from xfrm4_tunnel_encap_add. The reason for the
2349	* copy is that IP-TFS/AGGFRAG can have different functionality for how to set
2350	* the TOS/DSCP bits. Sets the protocol to a different value and doesn't do
2351	* anything with inner headers as they aren't pointing into a normal IP
2352	* singleton inner packet.
2353	*
2354	* Return: 0 on success or a negative error code on failure
2355	*/
2356	static int iptfs_encap_add_ipv4(struct xfrm_state *x, struct sk_buff *skb)
2357	{
2358	struct dst_entry *dst = skb_dst(skb);
2359	struct iphdr *top_iph;
2360
2361	skb_reset_inner_network_header(skb);
2362	skb_reset_inner_transport_header(skb);
2363
2364	skb_set_network_header(skb, offset: -(x->props.header_len - x->props.enc_hdr_len));
2365	skb->mac_header = skb->network_header + offsetof(struct iphdr, protocol);
2366	skb->transport_header = skb->network_header + sizeof(*top_iph);
2367
2368	top_iph = ip_hdr(skb);
2369	top_iph->ihl = 5;
2370	top_iph->version = 4;
2371	top_iph->protocol = IPPROTO_AGGFRAG;
2372
2373	/* As we have 0, fractional, 1 or N inner packets there's no obviously
2374	* correct DSCP mapping to inherit. ECN should be cleared per RFC9347
2375	* 3.1.
2376	*/
2377	top_iph->tos = 0;
2378
2379	top_iph->frag_off = htons(IP_DF);
2380	top_iph->ttl = ip4_dst_hoplimit(dst: xfrm_dst_child(dst));
2381	top_iph->saddr = x->props.saddr.a4;
2382	top_iph->daddr = x->id.daddr.a4;
2383	ip_select_ident(net: dev_net(dev: dst->dev), skb, NULL);
2384
2385	return 0;
2386	}
2387
2388	#if IS_ENABLED(CONFIG_IPV6)
2389	/**
2390	* iptfs_encap_add_ipv6() - add outer encaps
2391	* @x: xfrm state
2392	* @skb: the packet
2393	*
2394	* This was originally taken from xfrm6_tunnel_encap_add. The reason for the
2395	* copy is that IP-TFS/AGGFRAG can have different functionality for how to set
2396	* the flow label and TOS/DSCP bits. It also sets the protocol to a different
2397	* value and doesn't do anything with inner headers as they aren't pointing into
2398	* a normal IP singleton inner packet.
2399	*
2400	* Return: 0 on success or a negative error code on failure
2401	*/
2402	static int iptfs_encap_add_ipv6(struct xfrm_state *x, struct sk_buff *skb)
2403	{
2404	struct dst_entry *dst = skb_dst(skb);
2405	struct ipv6hdr *top_iph;
2406	int dsfield;
2407
2408	skb_reset_inner_network_header(skb);
2409	skb_reset_inner_transport_header(skb);
2410
2411	skb_set_network_header(skb, offset: -x->props.header_len + x->props.enc_hdr_len);
2412	skb->mac_header = skb->network_header + offsetof(struct ipv6hdr, nexthdr);
2413	skb->transport_header = skb->network_header + sizeof(*top_iph);
2414
2415	top_iph = ipv6_hdr(skb);
2416	top_iph->version = 6;
2417	top_iph->priority = 0;
2418	memset(top_iph->flow_lbl, 0, sizeof(top_iph->flow_lbl));
2419	top_iph->nexthdr = IPPROTO_AGGFRAG;
2420
2421	/* As we have 0, fractional, 1 or N inner packets there's no obviously
2422	* correct DSCP mapping to inherit. ECN should be cleared per RFC9347
2423	* 3.1.
2424	*/
2425	dsfield = 0;
2426	ipv6_change_dsfield(ipv6h: top_iph, mask: 0, value: dsfield);
2427
2428	top_iph->hop_limit = ip6_dst_hoplimit(dst: xfrm_dst_child(dst));
2429	top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
2430	top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
2431
2432	return 0;
2433	}
2434	#endif
2435
2436	/**
2437	* iptfs_prepare_output() - prepare the skb for output
2438	* @x: xfrm state
2439	* @skb: the packet
2440	*
2441	* Return: Error value, if 0 then skb values should be as follows:
2442	* - transport_header should point at ESP header
2443	* - network_header should point at Outer IP header
2444	* - mac_header should point at protocol/nexthdr of the outer IP
2445	*/
2446	static int iptfs_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
2447	{
2448	if (x->outer_mode.family == AF_INET)
2449	return iptfs_encap_add_ipv4(x, skb);
2450	if (x->outer_mode.family == AF_INET6) {
2451	#if IS_ENABLED(CONFIG_IPV6)
2452	return iptfs_encap_add_ipv6(x, skb);
2453	#else
2454	return -EAFNOSUPPORT;
2455	#endif
2456	}
2457	return -EOPNOTSUPP;
2458	}
2459
2460	/* ========================== */
2461	/* State Management Functions */
2462	/* ========================== */
2463
2464	/**
2465	* __iptfs_get_inner_mtu() - return inner MTU with no fragmentation.
2466	* @x: xfrm state.
2467	* @outer_mtu: the outer mtu
2468	*
2469	* Return: Correct MTU taking in to account the encap overhead.
2470	*/
2471	static u32 __iptfs_get_inner_mtu(struct xfrm_state *x, int outer_mtu)
2472	{
2473	struct crypto_aead *aead;
2474	u32 blksize;
2475
2476	aead = x->data;
2477	blksize = ALIGN(crypto_aead_blocksize(aead), 4);
2478	return ((outer_mtu - x->props.header_len - crypto_aead_authsize(tfm: aead)) &
2479	~(blksize - 1)) - 2;
2480	}
2481
2482	/**
2483	* iptfs_get_inner_mtu() - return the inner MTU for an IPTFS xfrm.
2484	* @x: xfrm state.
2485	* @outer_mtu: Outer MTU for the encapsulated packet.
2486	*
2487	* Return: Correct MTU taking in to account the encap overhead.
2488	*/
2489	static u32 iptfs_get_inner_mtu(struct xfrm_state *x, int outer_mtu)
2490	{
2491	struct xfrm_iptfs_data *xtfs = x->mode_data;
2492
2493	/* If not dont-frag we have no MTU */
2494	if (!xtfs->cfg.dont_frag)
2495	return x->outer_mode.family == AF_INET ? IP_MAX_MTU : IP6_MAX_MTU;
2496	return __iptfs_get_inner_mtu(x, outer_mtu);
2497	}
2498
2499	/**
2500	* iptfs_user_init() - initialize the SA with IPTFS options from netlink.
2501	* @net: the net data
2502	* @x: xfrm state
2503	* @attrs: netlink attributes
2504	* @extack: extack return data
2505	*
2506	* Return: 0 on success or a negative error code on failure
2507	*/
2508	static int iptfs_user_init(struct net *net, struct xfrm_state *x,
2509	struct nlattr **attrs,
2510	struct netlink_ext_ack *extack)
2511	{
2512	struct xfrm_iptfs_data *xtfs = x->mode_data;
2513	struct xfrm_iptfs_config *xc;
2514	u64 q;
2515
2516	xc = &xtfs->cfg;
2517	xc->max_queue_size = IPTFS_DEFAULT_MAX_QUEUE_SIZE;
2518	xc->reorder_win_size = IPTFS_DEFAULT_REORDER_WINDOW;
2519	xtfs->drop_time_ns = IPTFS_DEFAULT_DROP_TIME_USECS * NSECS_IN_USEC;
2520	xtfs->init_delay_ns = IPTFS_DEFAULT_INIT_DELAY_USECS * NSECS_IN_USEC;
2521
2522	if (attrs[XFRMA_IPTFS_DONT_FRAG])
2523	xc->dont_frag = true;
2524	if (attrs[XFRMA_IPTFS_REORDER_WINDOW])
2525	xc->reorder_win_size =
2526	nla_get_u16(nla: attrs[XFRMA_IPTFS_REORDER_WINDOW]);
2527	/* saved array is for saving 1..N seq nums from wantseq */
2528	if (xc->reorder_win_size) {
2529	xtfs->w_saved = kcalloc(xc->reorder_win_size,
2530	sizeof(*xtfs->w_saved), GFP_KERNEL);
2531	if (!xtfs->w_saved) {
2532	NL_SET_ERR_MSG(extack, "Cannot alloc reorder window");
2533	return -ENOMEM;
2534	}
2535	}
2536	if (attrs[XFRMA_IPTFS_PKT_SIZE]) {
2537	xc->pkt_size = nla_get_u32(nla: attrs[XFRMA_IPTFS_PKT_SIZE]);
2538	if (!xc->pkt_size) {
2539	xtfs->payload_mtu = 0;
2540	} else if (xc->pkt_size > x->props.header_len) {
2541	xtfs->payload_mtu = xc->pkt_size - x->props.header_len;
2542	} else {
2543	NL_SET_ERR_MSG(extack,
2544	"Packet size must be 0 or greater than IPTFS/ESP header length");
2545	return -EINVAL;
2546	}
2547	}
2548	if (attrs[XFRMA_IPTFS_MAX_QSIZE])
2549	xc->max_queue_size = nla_get_u32(nla: attrs[XFRMA_IPTFS_MAX_QSIZE]);
2550	if (attrs[XFRMA_IPTFS_DROP_TIME])
2551	xtfs->drop_time_ns =
2552	(u64)nla_get_u32(nla: attrs[XFRMA_IPTFS_DROP_TIME]) *
2553	NSECS_IN_USEC;
2554	if (attrs[XFRMA_IPTFS_INIT_DELAY])
2555	xtfs->init_delay_ns =
2556	(u64)nla_get_u32(nla: attrs[XFRMA_IPTFS_INIT_DELAY]) * NSECS_IN_USEC;
2557
2558	q = (u64)xc->max_queue_size * 95;
2559	do_div(q, 100);
2560	xtfs->ecn_queue_size = (u32)q;
2561
2562	return 0;
2563	}
2564
2565	static unsigned int iptfs_sa_len(const struct xfrm_state *x)
2566	{
2567	struct xfrm_iptfs_data *xtfs = x->mode_data;
2568	struct xfrm_iptfs_config *xc = &xtfs->cfg;
2569	unsigned int l = 0;
2570
2571	if (x->dir == XFRM_SA_DIR_IN) {
2572	l += nla_total_size(payload: sizeof(u32)); /* drop time usec */
2573	l += nla_total_size(payload: sizeof(xc->reorder_win_size));
2574	} else {
2575	if (xc->dont_frag)
2576	l += nla_total_size(payload: 0); /* dont-frag flag */
2577	l += nla_total_size(payload: sizeof(u32)); /* init delay usec */
2578	l += nla_total_size(payload: sizeof(xc->max_queue_size));
2579	l += nla_total_size(payload: sizeof(xc->pkt_size));
2580	}
2581
2582	return l;
2583	}
2584
2585	static int iptfs_copy_to_user(struct xfrm_state *x, struct sk_buff *skb)
2586	{
2587	struct xfrm_iptfs_data *xtfs = x->mode_data;
2588	struct xfrm_iptfs_config *xc = &xtfs->cfg;
2589	int ret = 0;
2590	u64 q;
2591
2592	if (x->dir == XFRM_SA_DIR_IN) {
2593	q = xtfs->drop_time_ns;
2594	do_div(q, NSECS_IN_USEC);
2595	ret = nla_put_u32(skb, attrtype: XFRMA_IPTFS_DROP_TIME, value: q);
2596	if (ret)
2597	return ret;
2598
2599	ret = nla_put_u16(skb, attrtype: XFRMA_IPTFS_REORDER_WINDOW,
2600	value: xc->reorder_win_size);
2601	} else {
2602	if (xc->dont_frag) {
2603	ret = nla_put_flag(skb, attrtype: XFRMA_IPTFS_DONT_FRAG);
2604	if (ret)
2605	return ret;
2606	}
2607
2608	q = xtfs->init_delay_ns;
2609	do_div(q, NSECS_IN_USEC);
2610	ret = nla_put_u32(skb, attrtype: XFRMA_IPTFS_INIT_DELAY, value: q);
2611	if (ret)
2612	return ret;
2613
2614	ret = nla_put_u32(skb, attrtype: XFRMA_IPTFS_MAX_QSIZE, value: xc->max_queue_size);
2615	if (ret)
2616	return ret;
2617
2618	ret = nla_put_u32(skb, attrtype: XFRMA_IPTFS_PKT_SIZE, value: xc->pkt_size);
2619	}
2620
2621	return ret;
2622	}
2623
2624	static void __iptfs_init_state(struct xfrm_state *x,
2625	struct xfrm_iptfs_data *xtfs)
2626	{
2627	__skb_queue_head_init(list: &xtfs->queue);
2628	hrtimer_setup(timer: &xtfs->iptfs_timer, function: iptfs_delay_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
2629
2630	spin_lock_init(&xtfs->drop_lock);
2631	hrtimer_setup(timer: &xtfs->drop_timer, function: iptfs_drop_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
2632
2633	/* Modify type (esp) adjustment values */
2634
2635	if (x->props.family == AF_INET)
2636	x->props.header_len += sizeof(struct iphdr) + sizeof(struct ip_iptfs_hdr);
2637	else if (x->props.family == AF_INET6)
2638	x->props.header_len += sizeof(struct ipv6hdr) + sizeof(struct ip_iptfs_hdr);
2639	x->props.enc_hdr_len = sizeof(struct ip_iptfs_hdr);
2640
2641	/* Always keep a module reference when x->mode_data is set */
2642	__module_get(module: x->mode_cbs->owner);
2643
2644	x->mode_data = xtfs;
2645	xtfs->x = x;
2646	}
2647
2648	static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig)
2649	{
2650	struct xfrm_iptfs_data *xtfs;
2651
2652	xtfs = kmemdup(orig->mode_data, sizeof(*xtfs), GFP_KERNEL);
2653	if (!xtfs)
2654	return -ENOMEM;
2655
2656	x->mode_data = xtfs;
2657	xtfs->x = x;
2658
2659	xtfs->ra_newskb = NULL;
2660	if (xtfs->cfg.reorder_win_size) {
2661	xtfs->w_saved = kcalloc(xtfs->cfg.reorder_win_size,
2662	sizeof(*xtfs->w_saved), GFP_KERNEL);
2663	if (!xtfs->w_saved) {
2664	kfree_sensitive(objp: xtfs);
2665	return -ENOMEM;
2666	}
2667	}
2668
2669	return 0;
2670	}
2671
2672	static int iptfs_init_state(struct xfrm_state *x)
2673	{
2674	struct xfrm_iptfs_data *xtfs;
2675
2676	if (x->mode_data) {
2677	/* We have arrived here from xfrm_state_clone() */
2678	xtfs = x->mode_data;
2679	} else {
2680	xtfs = kzalloc(sizeof(*xtfs), GFP_KERNEL);
2681	if (!xtfs)
2682	return -ENOMEM;
2683	}
2684
2685	__iptfs_init_state(x, xtfs);
2686
2687	return 0;
2688	}
2689
2690	static void iptfs_destroy_state(struct xfrm_state *x)
2691	{
2692	struct xfrm_iptfs_data *xtfs = x->mode_data;
2693	struct sk_buff_head list;
2694	struct skb_wseq *s, *se;
2695	struct sk_buff *skb;
2696
2697	if (!xtfs)
2698	return;
2699
2700	spin_lock_bh(lock: &xtfs->x->lock);
2701	hrtimer_cancel(timer: &xtfs->iptfs_timer);
2702	__skb_queue_head_init(list: &list);
2703	skb_queue_splice_init(list: &xtfs->queue, head: &list);
2704	spin_unlock_bh(lock: &xtfs->x->lock);
2705
2706	while ((skb = __skb_dequeue(list: &list)))
2707	kfree_skb(skb);
2708
2709	spin_lock_bh(lock: &xtfs->drop_lock);
2710	hrtimer_cancel(timer: &xtfs->drop_timer);
2711	spin_unlock_bh(lock: &xtfs->drop_lock);
2712
2713	if (xtfs->ra_newskb)
2714	kfree_skb(skb: xtfs->ra_newskb);
2715
2716	for (s = xtfs->w_saved, se = s + xtfs->w_savedlen; s < se; s++) {
2717	if (s->skb)
2718	kfree_skb(skb: s->skb);
2719	}
2720
2721	kfree_sensitive(objp: xtfs->w_saved);
2722	kfree_sensitive(objp: xtfs);
2723
2724	module_put(module: x->mode_cbs->owner);
2725	}
2726
2727	static const struct xfrm_mode_cbs iptfs_mode_cbs = {
2728	.owner = THIS_MODULE,
2729	.init_state = iptfs_init_state,
2730	.clone_state = iptfs_clone_state,
2731	.destroy_state = iptfs_destroy_state,
2732	.user_init = iptfs_user_init,
2733	.copy_to_user = iptfs_copy_to_user,
2734	.sa_len = iptfs_sa_len,
2735	.get_inner_mtu = iptfs_get_inner_mtu,
2736	.input = iptfs_input,
2737	.output = iptfs_output_collect,
2738	.prepare_output = iptfs_prepare_output,
2739	};
2740
2741	static int __init xfrm_iptfs_init(void)
2742	{
2743	int err;
2744
2745	pr_info("xfrm_iptfs: IPsec IP-TFS tunnel mode module\n");
2746
2747	err = xfrm_register_mode_cbs(XFRM_MODE_IPTFS, mode_cbs: &iptfs_mode_cbs);
2748	if (err < 0)
2749	pr_info("%s: can't register IP-TFS\n", __func__);
2750
2751	return err;
2752	}
2753
2754	static void __exit xfrm_iptfs_fini(void)
2755	{
2756	xfrm_unregister_mode_cbs(XFRM_MODE_IPTFS);
2757	}
2758
2759	module_init(xfrm_iptfs_init);
2760	module_exit(xfrm_iptfs_fini);
2761	MODULE_LICENSE("GPL");
2762	MODULE_DESCRIPTION("IP-TFS support for xfrm ipsec tunnels");
2763