1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | |
3 | #ifndef _NET_IPV6_GRO_H |
4 | #define _NET_IPV6_GRO_H |
5 | |
6 | #include <linux/indirect_call_wrapper.h> |
7 | #include <linux/ip.h> |
8 | #include <linux/ipv6.h> |
9 | #include <net/ip6_checksum.h> |
10 | #include <linux/skbuff.h> |
11 | #include <net/udp.h> |
12 | |
13 | struct napi_gro_cb { |
14 | union { |
15 | struct { |
16 | /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */ |
17 | void *frag0; |
18 | |
19 | /* Length of frag0. */ |
20 | unsigned int frag0_len; |
21 | }; |
22 | |
23 | struct { |
24 | /* used in skb_gro_receive() slow path */ |
25 | struct sk_buff *last; |
26 | |
27 | /* jiffies when first packet was created/queued */ |
28 | unsigned long age; |
29 | }; |
30 | }; |
31 | |
32 | /* This indicates where we are processing relative to skb->data. */ |
33 | int data_offset; |
34 | |
35 | /* This is non-zero if the packet cannot be merged with the new skb. */ |
36 | u16 flush; |
37 | |
38 | /* Save the IP ID here and check when we get to the transport layer */ |
39 | u16 flush_id; |
40 | |
41 | /* Number of segments aggregated. */ |
42 | u16 count; |
43 | |
44 | /* Used in ipv6_gro_receive() and foo-over-udp and esp-in-udp */ |
45 | u16 proto; |
46 | |
47 | /* Used in napi_gro_cb::free */ |
48 | #define NAPI_GRO_FREE 1 |
49 | #define NAPI_GRO_FREE_STOLEN_HEAD 2 |
50 | /* portion of the cb set to zero at every gro iteration */ |
51 | struct_group(zeroed, |
52 | |
53 | /* Start offset for remote checksum offload */ |
54 | u16 gro_remcsum_start; |
55 | |
56 | /* This is non-zero if the packet may be of the same flow. */ |
57 | u8 same_flow:1; |
58 | |
59 | /* Used in tunnel GRO receive */ |
60 | u8 encap_mark:1; |
61 | |
62 | /* GRO checksum is valid */ |
63 | u8 csum_valid:1; |
64 | |
65 | /* Number of checksums via CHECKSUM_UNNECESSARY */ |
66 | u8 csum_cnt:3; |
67 | |
68 | /* Free the skb? */ |
69 | u8 free:2; |
70 | |
71 | /* Used in foo-over-udp, set in udp[46]_gro_receive */ |
72 | u8 is_ipv6:1; |
73 | |
74 | /* Used in GRE, set in fou/gue_gro_receive */ |
75 | u8 is_fou:1; |
76 | |
77 | /* Used to determine if flush_id can be ignored */ |
78 | u8 is_atomic:1; |
79 | |
80 | /* Number of gro_receive callbacks this packet already went through */ |
81 | u8 recursion_counter:4; |
82 | |
83 | /* GRO is done by frag_list pointer chaining. */ |
84 | u8 is_flist:1; |
85 | ); |
86 | |
87 | /* used to support CHECKSUM_COMPLETE for tunneling protocols */ |
88 | __wsum csum; |
89 | }; |
90 | |
91 | #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) |
92 | |
93 | #define GRO_RECURSION_LIMIT 15 |
94 | static inline int gro_recursion_inc_test(struct sk_buff *skb) |
95 | { |
96 | return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; |
97 | } |
98 | |
99 | typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *); |
100 | static inline struct sk_buff *call_gro_receive(gro_receive_t cb, |
101 | struct list_head *head, |
102 | struct sk_buff *skb) |
103 | { |
104 | if (unlikely(gro_recursion_inc_test(skb))) { |
105 | NAPI_GRO_CB(skb)->flush |= 1; |
106 | return NULL; |
107 | } |
108 | |
109 | return cb(head, skb); |
110 | } |
111 | |
112 | typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *, |
113 | struct sk_buff *); |
114 | static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb, |
115 | struct sock *sk, |
116 | struct list_head *head, |
117 | struct sk_buff *skb) |
118 | { |
119 | if (unlikely(gro_recursion_inc_test(skb))) { |
120 | NAPI_GRO_CB(skb)->flush |= 1; |
121 | return NULL; |
122 | } |
123 | |
124 | return cb(sk, head, skb); |
125 | } |
126 | |
127 | static inline unsigned int skb_gro_offset(const struct sk_buff *skb) |
128 | { |
129 | return NAPI_GRO_CB(skb)->data_offset; |
130 | } |
131 | |
132 | static inline unsigned int skb_gro_len(const struct sk_buff *skb) |
133 | { |
134 | return skb->len - NAPI_GRO_CB(skb)->data_offset; |
135 | } |
136 | |
137 | static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) |
138 | { |
139 | NAPI_GRO_CB(skb)->data_offset += len; |
140 | } |
141 | |
142 | static inline void *(struct sk_buff *skb, |
143 | unsigned int offset) |
144 | { |
145 | return NAPI_GRO_CB(skb)->frag0 + offset; |
146 | } |
147 | |
148 | static inline int (struct sk_buff *skb, unsigned int hlen) |
149 | { |
150 | return NAPI_GRO_CB(skb)->frag0_len < hlen; |
151 | } |
152 | |
153 | static inline void skb_gro_frag0_invalidate(struct sk_buff *skb) |
154 | { |
155 | NAPI_GRO_CB(skb)->frag0 = NULL; |
156 | NAPI_GRO_CB(skb)->frag0_len = 0; |
157 | } |
158 | |
159 | static inline void *(struct sk_buff *skb, unsigned int hlen, |
160 | unsigned int offset) |
161 | { |
162 | if (!pskb_may_pull(skb, len: hlen)) |
163 | return NULL; |
164 | |
165 | skb_gro_frag0_invalidate(skb); |
166 | return skb->data + offset; |
167 | } |
168 | |
169 | static inline void *(struct sk_buff *skb, |
170 | unsigned int hlen, unsigned int offset) |
171 | { |
172 | void *ptr; |
173 | |
174 | ptr = skb_gro_header_fast(skb, offset); |
175 | if (skb_gro_header_hard(skb, hlen)) |
176 | ptr = skb_gro_header_slow(skb, hlen, offset); |
177 | return ptr; |
178 | } |
179 | |
180 | static inline void *(struct sk_buff *skb) |
181 | { |
182 | return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + |
183 | skb_network_offset(skb); |
184 | } |
185 | |
186 | static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto) |
187 | { |
188 | const struct iphdr *iph = skb_gro_network_header(skb); |
189 | |
190 | return csum_tcpudp_nofold(saddr: iph->saddr, daddr: iph->daddr, |
191 | len: skb_gro_len(skb), proto, sum: 0); |
192 | } |
193 | |
194 | static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, |
195 | const void *start, unsigned int len) |
196 | { |
197 | if (NAPI_GRO_CB(skb)->csum_valid) |
198 | NAPI_GRO_CB(skb)->csum = wsum_negate(val: csum_partial(buff: start, len, |
199 | sum: wsum_negate(NAPI_GRO_CB(skb)->csum))); |
200 | } |
201 | |
202 | /* GRO checksum functions. These are logical equivalents of the normal |
203 | * checksum functions (in skbuff.h) except that they operate on the GRO |
204 | * offsets and fields in sk_buff. |
205 | */ |
206 | |
207 | __sum16 __skb_gro_checksum_complete(struct sk_buff *skb); |
208 | |
209 | static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb) |
210 | { |
211 | return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb)); |
212 | } |
213 | |
214 | static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb, |
215 | bool zero_okay, |
216 | __sum16 check) |
217 | { |
218 | return ((skb->ip_summed != CHECKSUM_PARTIAL || |
219 | skb_checksum_start_offset(skb) < |
220 | skb_gro_offset(skb)) && |
221 | !skb_at_gro_remcsum_start(skb) && |
222 | NAPI_GRO_CB(skb)->csum_cnt == 0 && |
223 | (!zero_okay || check)); |
224 | } |
225 | |
226 | static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb, |
227 | __wsum psum) |
228 | { |
229 | if (NAPI_GRO_CB(skb)->csum_valid && |
230 | !csum_fold(sum: csum_add(csum: psum, NAPI_GRO_CB(skb)->csum))) |
231 | return 0; |
232 | |
233 | NAPI_GRO_CB(skb)->csum = psum; |
234 | |
235 | return __skb_gro_checksum_complete(skb); |
236 | } |
237 | |
238 | static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb) |
239 | { |
240 | if (NAPI_GRO_CB(skb)->csum_cnt > 0) { |
241 | /* Consume a checksum from CHECKSUM_UNNECESSARY */ |
242 | NAPI_GRO_CB(skb)->csum_cnt--; |
243 | } else { |
244 | /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we |
245 | * verified a new top level checksum or an encapsulated one |
246 | * during GRO. This saves work if we fallback to normal path. |
247 | */ |
248 | __skb_incr_checksum_unnecessary(skb); |
249 | } |
250 | } |
251 | |
252 | #define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \ |
253 | compute_pseudo) \ |
254 | ({ \ |
255 | __sum16 __ret = 0; \ |
256 | if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \ |
257 | __ret = __skb_gro_checksum_validate_complete(skb, \ |
258 | compute_pseudo(skb, proto)); \ |
259 | if (!__ret) \ |
260 | skb_gro_incr_csum_unnecessary(skb); \ |
261 | __ret; \ |
262 | }) |
263 | |
264 | #define skb_gro_checksum_validate(skb, proto, compute_pseudo) \ |
265 | __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo) |
266 | |
267 | #define skb_gro_checksum_validate_zero_check(skb, proto, check, \ |
268 | compute_pseudo) \ |
269 | __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo) |
270 | |
271 | #define skb_gro_checksum_simple_validate(skb) \ |
272 | __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo) |
273 | |
274 | static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb) |
275 | { |
276 | return (NAPI_GRO_CB(skb)->csum_cnt == 0 && |
277 | !NAPI_GRO_CB(skb)->csum_valid); |
278 | } |
279 | |
280 | static inline void __skb_gro_checksum_convert(struct sk_buff *skb, |
281 | __wsum pseudo) |
282 | { |
283 | NAPI_GRO_CB(skb)->csum = ~pseudo; |
284 | NAPI_GRO_CB(skb)->csum_valid = 1; |
285 | } |
286 | |
287 | #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \ |
288 | do { \ |
289 | if (__skb_gro_checksum_convert_check(skb)) \ |
290 | __skb_gro_checksum_convert(skb, \ |
291 | compute_pseudo(skb, proto)); \ |
292 | } while (0) |
293 | |
294 | struct gro_remcsum { |
295 | int offset; |
296 | __wsum delta; |
297 | }; |
298 | |
299 | static inline void skb_gro_remcsum_init(struct gro_remcsum *grc) |
300 | { |
301 | grc->offset = 0; |
302 | grc->delta = 0; |
303 | } |
304 | |
305 | static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, |
306 | unsigned int off, size_t hdrlen, |
307 | int start, int offset, |
308 | struct gro_remcsum *grc, |
309 | bool nopartial) |
310 | { |
311 | __wsum delta; |
312 | size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); |
313 | |
314 | BUG_ON(!NAPI_GRO_CB(skb)->csum_valid); |
315 | |
316 | if (!nopartial) { |
317 | NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start; |
318 | return ptr; |
319 | } |
320 | |
321 | ptr = skb_gro_header(skb, hlen: off + plen, offset: off); |
322 | if (!ptr) |
323 | return NULL; |
324 | |
325 | delta = remcsum_adjust(ptr: ptr + hdrlen, NAPI_GRO_CB(skb)->csum, |
326 | start, offset); |
327 | |
328 | /* Adjust skb->csum since we changed the packet */ |
329 | NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, addend: delta); |
330 | |
331 | grc->offset = off + hdrlen + offset; |
332 | grc->delta = delta; |
333 | |
334 | return ptr; |
335 | } |
336 | |
337 | static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, |
338 | struct gro_remcsum *grc) |
339 | { |
340 | void *ptr; |
341 | size_t plen = grc->offset + sizeof(u16); |
342 | |
343 | if (!grc->delta) |
344 | return; |
345 | |
346 | ptr = skb_gro_header(skb, hlen: plen, offset: grc->offset); |
347 | if (!ptr) |
348 | return; |
349 | |
350 | remcsum_unadjust(psum: (__sum16 *)ptr, delta: grc->delta); |
351 | } |
352 | |
353 | #ifdef CONFIG_XFRM_OFFLOAD |
354 | static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) |
355 | { |
356 | if (PTR_ERR(ptr: pp) != -EINPROGRESS) |
357 | NAPI_GRO_CB(skb)->flush |= flush; |
358 | } |
359 | static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, |
360 | struct sk_buff *pp, |
361 | int flush, |
362 | struct gro_remcsum *grc) |
363 | { |
364 | if (PTR_ERR(ptr: pp) != -EINPROGRESS) { |
365 | NAPI_GRO_CB(skb)->flush |= flush; |
366 | skb_gro_remcsum_cleanup(skb, grc); |
367 | skb->remcsum_offload = 0; |
368 | } |
369 | } |
370 | #else |
371 | static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) |
372 | { |
373 | NAPI_GRO_CB(skb)->flush |= flush; |
374 | } |
375 | static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, |
376 | struct sk_buff *pp, |
377 | int flush, |
378 | struct gro_remcsum *grc) |
379 | { |
380 | NAPI_GRO_CB(skb)->flush |= flush; |
381 | skb_gro_remcsum_cleanup(skb, grc); |
382 | skb->remcsum_offload = 0; |
383 | } |
384 | #endif |
385 | |
386 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, |
387 | struct sk_buff *)); |
388 | INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int)); |
389 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, |
390 | struct sk_buff *)); |
391 | INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); |
392 | |
393 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, |
394 | struct sk_buff *)); |
395 | INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int)); |
396 | |
397 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *, |
398 | struct sk_buff *)); |
399 | INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int)); |
400 | |
401 | #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \ |
402 | ({ \ |
403 | unlikely(gro_recursion_inc_test(skb)) ? \ |
404 | NAPI_GRO_CB(skb)->flush |= 1, NULL : \ |
405 | INDIRECT_CALL_INET(cb, f2, f1, head, skb); \ |
406 | }) |
407 | |
408 | struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, |
409 | struct udphdr *uh, struct sock *sk); |
410 | int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); |
411 | |
412 | static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) |
413 | { |
414 | struct udphdr *uh; |
415 | unsigned int hlen, off; |
416 | |
417 | off = skb_gro_offset(skb); |
418 | hlen = off + sizeof(*uh); |
419 | uh = skb_gro_header(skb, hlen, offset: off); |
420 | |
421 | return uh; |
422 | } |
423 | |
424 | static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) |
425 | { |
426 | const struct ipv6hdr *iph = skb_gro_network_header(skb); |
427 | |
428 | return ~csum_unfold(n: csum_ipv6_magic(saddr: &iph->saddr, daddr: &iph->daddr, |
429 | len: skb_gro_len(skb), proto, sum: 0)); |
430 | } |
431 | |
432 | int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); |
433 | |
434 | /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ |
435 | static inline void gro_normal_list(struct napi_struct *napi) |
436 | { |
437 | if (!napi->rx_count) |
438 | return; |
439 | netif_receive_skb_list_internal(head: &napi->rx_list); |
440 | INIT_LIST_HEAD(list: &napi->rx_list); |
441 | napi->rx_count = 0; |
442 | } |
443 | |
444 | /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, |
445 | * pass the whole batch up to the stack. |
446 | */ |
447 | static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) |
448 | { |
449 | list_add_tail(new: &skb->list, head: &napi->rx_list); |
450 | napi->rx_count += segs; |
451 | if (napi->rx_count >= READ_ONCE(gro_normal_batch)) |
452 | gro_normal_list(napi); |
453 | } |
454 | |
455 | /* This function is the alternative of 'inet_iif' and 'inet_sdif' |
456 | * functions in case we can not rely on fields of IPCB. |
457 | * |
458 | * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized. |
459 | * The caller must hold the RCU read lock. |
460 | */ |
461 | static inline void inet_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif) |
462 | { |
463 | *iif = inet_iif(skb) ?: skb->dev->ifindex; |
464 | *sdif = 0; |
465 | |
466 | #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) |
467 | if (netif_is_l3_slave(dev: skb->dev)) { |
468 | struct net_device *master = netdev_master_upper_dev_get_rcu(dev: skb->dev); |
469 | |
470 | *sdif = *iif; |
471 | *iif = master ? master->ifindex : 0; |
472 | } |
473 | #endif |
474 | } |
475 | |
476 | /* This function is the alternative of 'inet6_iif' and 'inet6_sdif' |
477 | * functions in case we can not rely on fields of IP6CB. |
478 | * |
479 | * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized. |
480 | * The caller must hold the RCU read lock. |
481 | */ |
482 | static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif) |
483 | { |
484 | /* using skb->dev->ifindex because skb_dst(skb) is not initialized */ |
485 | *iif = skb->dev->ifindex; |
486 | *sdif = 0; |
487 | |
488 | #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) |
489 | if (netif_is_l3_slave(dev: skb->dev)) { |
490 | struct net_device *master = netdev_master_upper_dev_get_rcu(dev: skb->dev); |
491 | |
492 | *sdif = *iif; |
493 | *iif = master ? master->ifindex : 0; |
494 | } |
495 | #endif |
496 | } |
497 | |
498 | extern struct list_head offload_base; |
499 | |
500 | #endif /* _NET_IPV6_GRO_H */ |
501 | |