1/* SPDX-License-Identifier: GPL-2.0-or-later */
2
3#ifndef _NET_GRO_H
4#define _NET_GRO_H
5
6#include <linux/indirect_call_wrapper.h>
7#include <linux/ip.h>
8#include <linux/ipv6.h>
9#include <net/ip6_checksum.h>
10#include <linux/skbuff.h>
11#include <net/udp.h>
12#include <net/hotdata.h>
13
14struct napi_gro_cb {
15 union {
16 struct {
17 /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
18 void *frag0;
19
20 /* Length of frag0. */
21 unsigned int frag0_len;
22 };
23
24 struct {
25 /* used in skb_gro_receive() slow path */
26 struct sk_buff *last;
27
28 /* jiffies when first packet was created/queued */
29 unsigned long age;
30 };
31 };
32
33 /* This indicates where we are processing relative to skb->data. */
34 int data_offset;
35
36 /* This is non-zero if the packet cannot be merged with the new skb. */
37 u16 flush;
38
39 /* Save the IP ID here and check when we get to the transport layer */
40 u16 flush_id;
41
42 /* Number of segments aggregated. */
43 u16 count;
44
45 /* Used in ipv6_gro_receive() and foo-over-udp and esp-in-udp */
46 u16 proto;
47
48/* Used in napi_gro_cb::free */
49#define NAPI_GRO_FREE 1
50#define NAPI_GRO_FREE_STOLEN_HEAD 2
51 /* portion of the cb set to zero at every gro iteration */
52 struct_group(zeroed,
53
54 /* Start offset for remote checksum offload */
55 u16 gro_remcsum_start;
56
57 /* This is non-zero if the packet may be of the same flow. */
58 u8 same_flow:1;
59
60 /* Used in tunnel GRO receive */
61 u8 encap_mark:1;
62
63 /* GRO checksum is valid */
64 u8 csum_valid:1;
65
66 /* Number of checksums via CHECKSUM_UNNECESSARY */
67 u8 csum_cnt:3;
68
69 /* Free the skb? */
70 u8 free:2;
71
72 /* Used in foo-over-udp, set in udp[46]_gro_receive */
73 u8 is_ipv6:1;
74
75 /* Used in GRE, set in fou/gue_gro_receive */
76 u8 is_fou:1;
77
78 /* Used to determine if flush_id can be ignored */
79 u8 is_atomic:1;
80
81 /* Number of gro_receive callbacks this packet already went through */
82 u8 recursion_counter:4;
83
84 /* GRO is done by frag_list pointer chaining. */
85 u8 is_flist:1;
86 );
87
88 /* used to support CHECKSUM_COMPLETE for tunneling protocols */
89 __wsum csum;
90};
91
92#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
93
94#define GRO_RECURSION_LIMIT 15
95static inline int gro_recursion_inc_test(struct sk_buff *skb)
96{
97 return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
98}
99
100typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);
101static inline struct sk_buff *call_gro_receive(gro_receive_t cb,
102 struct list_head *head,
103 struct sk_buff *skb)
104{
105 if (unlikely(gro_recursion_inc_test(skb))) {
106 NAPI_GRO_CB(skb)->flush |= 1;
107 return NULL;
108 }
109
110 return cb(head, skb);
111}
112
113typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,
114 struct sk_buff *);
115static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,
116 struct sock *sk,
117 struct list_head *head,
118 struct sk_buff *skb)
119{
120 if (unlikely(gro_recursion_inc_test(skb))) {
121 NAPI_GRO_CB(skb)->flush |= 1;
122 return NULL;
123 }
124
125 return cb(sk, head, skb);
126}
127
128static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
129{
130 return NAPI_GRO_CB(skb)->data_offset;
131}
132
133static inline unsigned int skb_gro_len(const struct sk_buff *skb)
134{
135 return skb->len - NAPI_GRO_CB(skb)->data_offset;
136}
137
138static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
139{
140 NAPI_GRO_CB(skb)->data_offset += len;
141}
142
143static inline void *skb_gro_header_fast(const struct sk_buff *skb,
144 unsigned int offset)
145{
146 return NAPI_GRO_CB(skb)->frag0 + offset;
147}
148
149static inline bool skb_gro_may_pull(const struct sk_buff *skb,
150 unsigned int hlen)
151{
152 return likely(hlen <= NAPI_GRO_CB(skb)->frag0_len);
153}
154
155static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
156 unsigned int offset)
157{
158 if (!pskb_may_pull(skb, len: hlen))
159 return NULL;
160
161 return skb->data + offset;
162}
163
164static inline void *skb_gro_header(struct sk_buff *skb, unsigned int hlen,
165 unsigned int offset)
166{
167 void *ptr;
168
169 ptr = skb_gro_header_fast(skb, offset);
170 if (!skb_gro_may_pull(skb, hlen))
171 ptr = skb_gro_header_slow(skb, hlen, offset);
172 return ptr;
173}
174
175static inline void *skb_gro_network_header(const struct sk_buff *skb)
176{
177 if (skb_gro_may_pull(skb, hlen: skb_gro_offset(skb)))
178 return skb_gro_header_fast(skb, offset: skb_network_offset(skb));
179
180 return skb_network_header(skb);
181}
182
183static inline __wsum inet_gro_compute_pseudo(const struct sk_buff *skb,
184 int proto)
185{
186 const struct iphdr *iph = skb_gro_network_header(skb);
187
188 return csum_tcpudp_nofold(saddr: iph->saddr, daddr: iph->daddr,
189 len: skb_gro_len(skb), proto, sum: 0);
190}
191
192static inline void skb_gro_postpull_rcsum(struct sk_buff *skb,
193 const void *start, unsigned int len)
194{
195 if (NAPI_GRO_CB(skb)->csum_valid)
196 NAPI_GRO_CB(skb)->csum = wsum_negate(val: csum_partial(buff: start, len,
197 sum: wsum_negate(NAPI_GRO_CB(skb)->csum)));
198}
199
200/* GRO checksum functions. These are logical equivalents of the normal
201 * checksum functions (in skbuff.h) except that they operate on the GRO
202 * offsets and fields in sk_buff.
203 */
204
205__sum16 __skb_gro_checksum_complete(struct sk_buff *skb);
206
207static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb)
208{
209 return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));
210}
211
212static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb,
213 bool zero_okay,
214 __sum16 check)
215{
216 return ((skb->ip_summed != CHECKSUM_PARTIAL ||
217 skb_checksum_start_offset(skb) <
218 skb_gro_offset(skb)) &&
219 !skb_at_gro_remcsum_start(skb) &&
220 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
221 (!zero_okay || check));
222}
223
224static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb,
225 __wsum psum)
226{
227 if (NAPI_GRO_CB(skb)->csum_valid &&
228 !csum_fold(sum: csum_add(csum: psum, NAPI_GRO_CB(skb)->csum)))
229 return 0;
230
231 NAPI_GRO_CB(skb)->csum = psum;
232
233 return __skb_gro_checksum_complete(skb);
234}
235
236static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb)
237{
238 if (NAPI_GRO_CB(skb)->csum_cnt > 0) {
239 /* Consume a checksum from CHECKSUM_UNNECESSARY */
240 NAPI_GRO_CB(skb)->csum_cnt--;
241 } else {
242 /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we
243 * verified a new top level checksum or an encapsulated one
244 * during GRO. This saves work if we fallback to normal path.
245 */
246 __skb_incr_checksum_unnecessary(skb);
247 }
248}
249
250#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \
251 compute_pseudo) \
252({ \
253 __sum16 __ret = 0; \
254 if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \
255 __ret = __skb_gro_checksum_validate_complete(skb, \
256 compute_pseudo(skb, proto)); \
257 if (!__ret) \
258 skb_gro_incr_csum_unnecessary(skb); \
259 __ret; \
260})
261
262#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \
263 __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo)
264
265#define skb_gro_checksum_validate_zero_check(skb, proto, check, \
266 compute_pseudo) \
267 __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo)
268
269#define skb_gro_checksum_simple_validate(skb) \
270 __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo)
271
272static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb)
273{
274 return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&
275 !NAPI_GRO_CB(skb)->csum_valid);
276}
277
278static inline void __skb_gro_checksum_convert(struct sk_buff *skb,
279 __wsum pseudo)
280{
281 NAPI_GRO_CB(skb)->csum = ~pseudo;
282 NAPI_GRO_CB(skb)->csum_valid = 1;
283}
284
285#define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \
286do { \
287 if (__skb_gro_checksum_convert_check(skb)) \
288 __skb_gro_checksum_convert(skb, \
289 compute_pseudo(skb, proto)); \
290} while (0)
291
292struct gro_remcsum {
293 int offset;
294 __wsum delta;
295};
296
297static inline void skb_gro_remcsum_init(struct gro_remcsum *grc)
298{
299 grc->offset = 0;
300 grc->delta = 0;
301}
302
303static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
304 unsigned int off, size_t hdrlen,
305 int start, int offset,
306 struct gro_remcsum *grc,
307 bool nopartial)
308{
309 __wsum delta;
310 size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start);
311
312 BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);
313
314 if (!nopartial) {
315 NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;
316 return ptr;
317 }
318
319 ptr = skb_gro_header(skb, hlen: off + plen, offset: off);
320 if (!ptr)
321 return NULL;
322
323 delta = remcsum_adjust(ptr: ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
324 start, offset);
325
326 /* Adjust skb->csum since we changed the packet */
327 NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, addend: delta);
328
329 grc->offset = off + hdrlen + offset;
330 grc->delta = delta;
331
332 return ptr;
333}
334
335static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
336 struct gro_remcsum *grc)
337{
338 void *ptr;
339 size_t plen = grc->offset + sizeof(u16);
340
341 if (!grc->delta)
342 return;
343
344 ptr = skb_gro_header(skb, hlen: plen, offset: grc->offset);
345 if (!ptr)
346 return;
347
348 remcsum_unadjust(psum: (__sum16 *)ptr, delta: grc->delta);
349}
350
351#ifdef CONFIG_XFRM_OFFLOAD
352static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
353{
354 if (PTR_ERR(ptr: pp) != -EINPROGRESS)
355 NAPI_GRO_CB(skb)->flush |= flush;
356}
357static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
358 struct sk_buff *pp,
359 int flush,
360 struct gro_remcsum *grc)
361{
362 if (PTR_ERR(ptr: pp) != -EINPROGRESS) {
363 NAPI_GRO_CB(skb)->flush |= flush;
364 skb_gro_remcsum_cleanup(skb, grc);
365 skb->remcsum_offload = 0;
366 }
367}
368#else
369static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
370{
371 NAPI_GRO_CB(skb)->flush |= flush;
372}
373static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb,
374 struct sk_buff *pp,
375 int flush,
376 struct gro_remcsum *grc)
377{
378 NAPI_GRO_CB(skb)->flush |= flush;
379 skb_gro_remcsum_cleanup(skb, grc);
380 skb->remcsum_offload = 0;
381}
382#endif
383
384INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,
385 struct sk_buff *));
386INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
387INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
388 struct sk_buff *));
389INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
390
391INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *,
392 struct sk_buff *));
393INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int));
394
395INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *,
396 struct sk_buff *));
397INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int));
398
399#define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \
400({ \
401 unlikely(gro_recursion_inc_test(skb)) ? \
402 NAPI_GRO_CB(skb)->flush |= 1, NULL : \
403 INDIRECT_CALL_INET(cb, f2, f1, head, skb); \
404})
405
406struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
407 struct udphdr *uh, struct sock *sk);
408int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
409
410static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
411{
412 struct udphdr *uh;
413 unsigned int hlen, off;
414
415 off = skb_gro_offset(skb);
416 hlen = off + sizeof(*uh);
417 uh = skb_gro_header(skb, hlen, offset: off);
418
419 return uh;
420}
421
422static inline __wsum ip6_gro_compute_pseudo(const struct sk_buff *skb,
423 int proto)
424{
425 const struct ipv6hdr *iph = skb_gro_network_header(skb);
426
427 return ~csum_unfold(n: csum_ipv6_magic(saddr: &iph->saddr, daddr: &iph->daddr,
428 len: skb_gro_len(skb), proto, sum: 0));
429}
430
431int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
432
433/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
434static inline void gro_normal_list(struct napi_struct *napi)
435{
436 if (!napi->rx_count)
437 return;
438 netif_receive_skb_list_internal(head: &napi->rx_list);
439 INIT_LIST_HEAD(list: &napi->rx_list);
440 napi->rx_count = 0;
441}
442
443/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
444 * pass the whole batch up to the stack.
445 */
446static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
447{
448 list_add_tail(new: &skb->list, head: &napi->rx_list);
449 napi->rx_count += segs;
450 if (napi->rx_count >= READ_ONCE(net_hotdata.gro_normal_batch))
451 gro_normal_list(napi);
452}
453
454/* This function is the alternative of 'inet_iif' and 'inet_sdif'
455 * functions in case we can not rely on fields of IPCB.
456 *
457 * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized.
458 * The caller must hold the RCU read lock.
459 */
460static inline void inet_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif)
461{
462 *iif = inet_iif(skb) ?: skb->dev->ifindex;
463 *sdif = 0;
464
465#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
466 if (netif_is_l3_slave(dev: skb->dev)) {
467 struct net_device *master = netdev_master_upper_dev_get_rcu(dev: skb->dev);
468
469 *sdif = *iif;
470 *iif = master ? master->ifindex : 0;
471 }
472#endif
473}
474
475/* This function is the alternative of 'inet6_iif' and 'inet6_sdif'
476 * functions in case we can not rely on fields of IP6CB.
477 *
478 * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized.
479 * The caller must hold the RCU read lock.
480 */
481static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif)
482{
483 /* using skb->dev->ifindex because skb_dst(skb) is not initialized */
484 *iif = skb->dev->ifindex;
485 *sdif = 0;
486
487#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
488 if (netif_is_l3_slave(dev: skb->dev)) {
489 struct net_device *master = netdev_master_upper_dev_get_rcu(dev: skb->dev);
490
491 *sdif = *iif;
492 *iif = master ? master->ifindex : 0;
493 }
494#endif
495}
496
497struct packet_offload *gro_find_receive_by_type(__be16 type);
498struct packet_offload *gro_find_complete_by_type(__be16 type);
499
500#endif /* _NET_GRO_H */
501

source code of linux/include/net/gro.h