1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * IPV4 GSO/GRO offload support
4 * Linux INET implementation
5 *
6 * UDPv4 GSO support
7 */
8
9#include <linux/skbuff.h>
10#include <net/gro.h>
11#include <net/gso.h>
12#include <net/udp.h>
13#include <net/protocol.h>
14#include <net/inet_common.h>
15#include <net/udp_tunnel.h>
16
17#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
18
19/*
20 * Dummy GRO tunnel callback, exists mainly to avoid dangling/NULL
21 * values for the udp tunnel static call.
22 */
23static struct sk_buff *dummy_gro_rcv(struct sock *sk,
24 struct list_head *head,
25 struct sk_buff *skb)
26{
27 NAPI_GRO_CB(skb)->flush = 1;
28 return NULL;
29}
30
31typedef struct sk_buff *(*udp_tunnel_gro_rcv_t)(struct sock *sk,
32 struct list_head *head,
33 struct sk_buff *skb);
34
35struct udp_tunnel_type_entry {
36 udp_tunnel_gro_rcv_t gro_receive;
37 refcount_t count;
38};
39
40#define UDP_MAX_TUNNEL_TYPES (IS_ENABLED(CONFIG_GENEVE) + \
41 IS_ENABLED(CONFIG_VXLAN) * 2 + \
42 IS_ENABLED(CONFIG_NET_FOU) * 2 + \
43 IS_ENABLED(CONFIG_XFRM) * 2)
44
45DEFINE_STATIC_CALL(udp_tunnel_gro_rcv, dummy_gro_rcv);
46static DEFINE_STATIC_KEY_FALSE(udp_tunnel_static_call);
47static DEFINE_MUTEX(udp_tunnel_gro_type_lock);
48static struct udp_tunnel_type_entry udp_tunnel_gro_types[UDP_MAX_TUNNEL_TYPES];
49static unsigned int udp_tunnel_gro_type_nr;
50static DEFINE_SPINLOCK(udp_tunnel_gro_lock);
51
52void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add)
53{
54 bool is_ipv6 = sk->sk_family == AF_INET6;
55 struct udp_sock *tup, *up = udp_sk(sk);
56 struct udp_tunnel_gro *udp_tunnel_gro;
57
58 spin_lock(lock: &udp_tunnel_gro_lock);
59 udp_tunnel_gro = &net->ipv4.udp_tunnel_gro[is_ipv6];
60 if (add)
61 hlist_add_head(n: &up->tunnel_list, h: &udp_tunnel_gro->list);
62 else if (up->tunnel_list.pprev)
63 hlist_del_init(n: &up->tunnel_list);
64
65 if (udp_tunnel_gro->list.first &&
66 !udp_tunnel_gro->list.first->next) {
67 tup = hlist_entry(udp_tunnel_gro->list.first, struct udp_sock,
68 tunnel_list);
69
70 rcu_assign_pointer(udp_tunnel_gro->sk, (struct sock *)tup);
71 } else {
72 RCU_INIT_POINTER(udp_tunnel_gro->sk, NULL);
73 }
74
75 spin_unlock(lock: &udp_tunnel_gro_lock);
76}
77EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_lookup);
78
79void udp_tunnel_update_gro_rcv(struct sock *sk, bool add)
80{
81 struct udp_tunnel_type_entry *cur = NULL;
82 struct udp_sock *up = udp_sk(sk);
83 int i, old_gro_type_nr;
84
85 if (!UDP_MAX_TUNNEL_TYPES || !up->gro_receive)
86 return;
87
88 mutex_lock(&udp_tunnel_gro_type_lock);
89
90 /* Check if the static call is permanently disabled. */
91 if (udp_tunnel_gro_type_nr > UDP_MAX_TUNNEL_TYPES)
92 goto out;
93
94 for (i = 0; i < udp_tunnel_gro_type_nr; i++)
95 if (udp_tunnel_gro_types[i].gro_receive == up->gro_receive)
96 cur = &udp_tunnel_gro_types[i];
97
98 old_gro_type_nr = udp_tunnel_gro_type_nr;
99 if (add) {
100 /*
101 * Update the matching entry, if found, or add a new one
102 * if needed
103 */
104 if (cur) {
105 refcount_inc(r: &cur->count);
106 goto out;
107 }
108
109 if (unlikely(udp_tunnel_gro_type_nr == UDP_MAX_TUNNEL_TYPES)) {
110 pr_err_once("Too many UDP tunnel types, please increase UDP_MAX_TUNNEL_TYPES\n");
111 /* Ensure static call will never be enabled */
112 udp_tunnel_gro_type_nr = UDP_MAX_TUNNEL_TYPES + 1;
113 } else {
114 cur = &udp_tunnel_gro_types[udp_tunnel_gro_type_nr++];
115 refcount_set(r: &cur->count, n: 1);
116 cur->gro_receive = up->gro_receive;
117 }
118 } else {
119 /*
120 * The stack cleanups only successfully added tunnel, the
121 * lookup on removal should never fail.
122 */
123 if (WARN_ON_ONCE(!cur))
124 goto out;
125
126 if (!refcount_dec_and_test(r: &cur->count))
127 goto out;
128
129 /* Avoid gaps, so that the enable tunnel has always id 0 */
130 *cur = udp_tunnel_gro_types[--udp_tunnel_gro_type_nr];
131 }
132
133 if (udp_tunnel_gro_type_nr == 1) {
134 static_call_update(udp_tunnel_gro_rcv,
135 udp_tunnel_gro_types[0].gro_receive);
136 static_branch_enable(&udp_tunnel_static_call);
137 } else if (old_gro_type_nr == 1) {
138 static_branch_disable(&udp_tunnel_static_call);
139 static_call_update(udp_tunnel_gro_rcv, dummy_gro_rcv);
140 }
141
142out:
143 mutex_unlock(lock: &udp_tunnel_gro_type_lock);
144}
145EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_rcv);
146
147static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
148 struct list_head *head,
149 struct sk_buff *skb)
150{
151 if (static_branch_likely(&udp_tunnel_static_call)) {
152 if (unlikely(gro_recursion_inc_test(skb))) {
153 NAPI_GRO_CB(skb)->flush |= 1;
154 return NULL;
155 }
156 return static_call(udp_tunnel_gro_rcv)(sk, head, skb);
157 }
158 return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
159}
160
161#else
162
163static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
164 struct list_head *head,
165 struct sk_buff *skb)
166{
167 return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
168}
169
170#endif
171
172static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
173 netdev_features_t features,
174 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
175 netdev_features_t features),
176 __be16 new_protocol, bool is_ipv6)
177{
178 int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
179 bool remcsum, need_csum, offload_csum, gso_partial;
180 struct sk_buff *segs = ERR_PTR(error: -EINVAL);
181 struct udphdr *uh = udp_hdr(skb);
182 u16 mac_offset = skb->mac_header;
183 __be16 protocol = skb->protocol;
184 u16 mac_len = skb->mac_len;
185 int udp_offset, outer_hlen;
186 __wsum partial;
187 bool need_ipsec;
188
189 if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
190 goto out;
191
192 /* Adjust partial header checksum to negate old length.
193 * We cannot rely on the value contained in uh->len as it is
194 * possible that the actual value exceeds the boundaries of the
195 * 16 bit length field due to the header being added outside of an
196 * IP or IPv6 frame that was already limited to 64K - 1.
197 */
198 if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)
199 partial = (__force __wsum)uh->len;
200 else
201 partial = (__force __wsum)htonl(skb->len);
202 partial = csum_sub(csum: csum_unfold(n: uh->check), addend: partial);
203
204 /* setup inner skb. */
205 skb->encapsulation = 0;
206 SKB_GSO_CB(skb)->encap_level = 0;
207 __skb_pull(skb, len: tnl_hlen);
208 skb_reset_mac_header(skb);
209 skb_set_network_header(skb, offset: skb_inner_network_offset(skb));
210 skb_set_transport_header(skb, offset: skb_inner_transport_offset(skb));
211 skb->mac_len = skb_inner_network_offset(skb);
212 skb->protocol = new_protocol;
213
214 need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
215 skb->encap_hdr_csum = need_csum;
216
217 remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM);
218 skb->remcsum_offload = remcsum;
219
220 need_ipsec = (skb_dst(skb) && dst_xfrm(dst: skb_dst(skb))) || skb_sec_path(skb);
221 /* Try to offload checksum if possible */
222 offload_csum = !!(need_csum &&
223 !need_ipsec &&
224 (skb->dev->features &
225 (is_ipv6 ? (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM) :
226 (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM))));
227
228 features &= skb->dev->hw_enc_features;
229 if (need_csum)
230 features &= ~NETIF_F_SCTP_CRC;
231
232 /* The only checksum offload we care about from here on out is the
233 * outer one so strip the existing checksum feature flags and
234 * instead set the flag based on our outer checksum offload value.
235 */
236 if (remcsum) {
237 features &= ~NETIF_F_CSUM_MASK;
238 if (!need_csum || offload_csum)
239 features |= NETIF_F_HW_CSUM;
240 }
241
242 /* segment inner packet. */
243 segs = gso_inner_segment(skb, features);
244 if (IS_ERR_OR_NULL(ptr: segs)) {
245 skb_gso_error_unwind(skb, protocol, pulled_hlen: tnl_hlen, mac_offset,
246 mac_len);
247 goto out;
248 }
249
250 gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
251
252 outer_hlen = skb_tnl_header_len(inner_skb: skb);
253 udp_offset = outer_hlen - tnl_hlen;
254 skb = segs;
255 do {
256 unsigned int len;
257
258 if (remcsum)
259 skb->ip_summed = CHECKSUM_NONE;
260
261 /* Set up inner headers if we are offloading inner checksum */
262 if (skb->ip_summed == CHECKSUM_PARTIAL) {
263 skb_reset_inner_headers(skb);
264 skb->encapsulation = 1;
265 }
266
267 skb->mac_len = mac_len;
268 skb->protocol = protocol;
269
270 __skb_push(skb, len: outer_hlen);
271 skb_reset_mac_header(skb);
272 skb_set_network_header(skb, offset: mac_len);
273 skb_set_transport_header(skb, offset: udp_offset);
274 len = skb->len - udp_offset;
275 uh = udp_hdr(skb);
276
277 /* If we are only performing partial GSO the inner header
278 * will be using a length value equal to only one MSS sized
279 * segment instead of the entire frame.
280 */
281 if (gso_partial && skb_is_gso(skb)) {
282 uh->len = htons(skb_shinfo(skb)->gso_size +
283 SKB_GSO_CB(skb)->data_offset +
284 skb->head - (unsigned char *)uh);
285 } else {
286 uh->len = htons(len);
287 }
288
289 if (!need_csum)
290 continue;
291
292 uh->check = ~csum_fold(csum: csum_add(csum: partial,
293 addend: (__force __wsum)htonl(len)));
294
295 if (skb->encapsulation || !offload_csum) {
296 uh->check = gso_make_checksum(skb, res: ~uh->check);
297 if (uh->check == 0)
298 uh->check = CSUM_MANGLED_0;
299 } else {
300 skb->ip_summed = CHECKSUM_PARTIAL;
301 skb->csum_start = skb_transport_header(skb) - skb->head;
302 skb->csum_offset = offsetof(struct udphdr, check);
303 }
304 } while ((skb = skb->next));
305out:
306 return segs;
307}
308
309struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
310 netdev_features_t features,
311 bool is_ipv6)
312{
313 const struct net_offload __rcu **offloads;
314 __be16 protocol = skb->protocol;
315 const struct net_offload *ops;
316 struct sk_buff *segs = ERR_PTR(error: -EINVAL);
317 struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
318 netdev_features_t features);
319
320 rcu_read_lock();
321
322 switch (skb->inner_protocol_type) {
323 case ENCAP_TYPE_ETHER:
324 protocol = skb->inner_protocol;
325 gso_inner_segment = skb_mac_gso_segment;
326 break;
327 case ENCAP_TYPE_IPPROTO:
328 offloads = is_ipv6 ? inet6_offloads : inet_offloads;
329 ops = rcu_dereference(offloads[skb->inner_ipproto]);
330 if (!ops || !ops->callbacks.gso_segment)
331 goto out_unlock;
332 gso_inner_segment = ops->callbacks.gso_segment;
333 break;
334 default:
335 goto out_unlock;
336 }
337
338 segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,
339 new_protocol: protocol, is_ipv6);
340
341out_unlock:
342 rcu_read_unlock();
343
344 return segs;
345}
346EXPORT_SYMBOL(skb_udp_tunnel_segment);
347
348static void __udpv4_gso_segment_csum(struct sk_buff *seg,
349 __be32 *oldip, __be32 *newip,
350 __be16 *oldport, __be16 *newport)
351{
352 struct udphdr *uh;
353 struct iphdr *iph;
354
355 if (*oldip == *newip && *oldport == *newport)
356 return;
357
358 uh = udp_hdr(skb: seg);
359 iph = ip_hdr(skb: seg);
360
361 if (uh->check) {
362 inet_proto_csum_replace4(sum: &uh->check, skb: seg, from: *oldip, to: *newip,
363 pseudohdr: true);
364 inet_proto_csum_replace2(sum: &uh->check, skb: seg, from: *oldport, to: *newport,
365 pseudohdr: false);
366 if (!uh->check)
367 uh->check = CSUM_MANGLED_0;
368 }
369 *oldport = *newport;
370
371 csum_replace4(sum: &iph->check, from: *oldip, to: *newip);
372 *oldip = *newip;
373}
374
375static struct sk_buff *__udpv4_gso_segment_list_csum(struct sk_buff *segs)
376{
377 struct sk_buff *seg;
378 struct udphdr *uh, *uh2;
379 struct iphdr *iph, *iph2;
380
381 seg = segs;
382 uh = udp_hdr(skb: seg);
383 iph = ip_hdr(skb: seg);
384
385 if ((udp_hdr(skb: seg)->dest == udp_hdr(skb: seg->next)->dest) &&
386 (udp_hdr(skb: seg)->source == udp_hdr(skb: seg->next)->source) &&
387 (ip_hdr(skb: seg)->daddr == ip_hdr(skb: seg->next)->daddr) &&
388 (ip_hdr(skb: seg)->saddr == ip_hdr(skb: seg->next)->saddr))
389 return segs;
390
391 while ((seg = seg->next)) {
392 uh2 = udp_hdr(skb: seg);
393 iph2 = ip_hdr(skb: seg);
394
395 __udpv4_gso_segment_csum(seg,
396 oldip: &iph2->saddr, newip: &iph->saddr,
397 oldport: &uh2->source, newport: &uh->source);
398 __udpv4_gso_segment_csum(seg,
399 oldip: &iph2->daddr, newip: &iph->daddr,
400 oldport: &uh2->dest, newport: &uh->dest);
401 }
402
403 return segs;
404}
405
406static void __udpv6_gso_segment_csum(struct sk_buff *seg,
407 struct in6_addr *oldip,
408 const struct in6_addr *newip,
409 __be16 *oldport, __be16 newport)
410{
411 struct udphdr *uh = udp_hdr(skb: seg);
412
413 if (ipv6_addr_equal(a1: oldip, a2: newip) && *oldport == newport)
414 return;
415
416 if (uh->check) {
417 inet_proto_csum_replace16(sum: &uh->check, skb: seg, from: oldip->s6_addr32,
418 to: newip->s6_addr32, pseudohdr: true);
419
420 inet_proto_csum_replace2(sum: &uh->check, skb: seg, from: *oldport, to: newport,
421 pseudohdr: false);
422 if (!uh->check)
423 uh->check = CSUM_MANGLED_0;
424 }
425
426 *oldip = *newip;
427 *oldport = newport;
428}
429
430static struct sk_buff *__udpv6_gso_segment_list_csum(struct sk_buff *segs)
431{
432 const struct ipv6hdr *iph;
433 const struct udphdr *uh;
434 struct ipv6hdr *iph2;
435 struct sk_buff *seg;
436 struct udphdr *uh2;
437
438 seg = segs;
439 uh = udp_hdr(skb: seg);
440 iph = ipv6_hdr(skb: seg);
441 uh2 = udp_hdr(skb: seg->next);
442 iph2 = ipv6_hdr(skb: seg->next);
443
444 if (!(*(const u32 *)&uh->source ^ *(const u32 *)&uh2->source) &&
445 ipv6_addr_equal(a1: &iph->saddr, a2: &iph2->saddr) &&
446 ipv6_addr_equal(a1: &iph->daddr, a2: &iph2->daddr))
447 return segs;
448
449 while ((seg = seg->next)) {
450 uh2 = udp_hdr(skb: seg);
451 iph2 = ipv6_hdr(skb: seg);
452
453 __udpv6_gso_segment_csum(seg, oldip: &iph2->saddr, newip: &iph->saddr,
454 oldport: &uh2->source, newport: uh->source);
455 __udpv6_gso_segment_csum(seg, oldip: &iph2->daddr, newip: &iph->daddr,
456 oldport: &uh2->dest, newport: uh->dest);
457 }
458
459 return segs;
460}
461
462static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,
463 netdev_features_t features,
464 bool is_ipv6)
465{
466 unsigned int mss = skb_shinfo(skb)->gso_size;
467
468 skb = skb_segment_list(skb, features, offset: skb_mac_header_len(skb));
469 if (IS_ERR(ptr: skb))
470 return skb;
471
472 udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);
473
474 if (is_ipv6)
475 return __udpv6_gso_segment_list_csum(segs: skb);
476 else
477 return __udpv4_gso_segment_list_csum(segs: skb);
478}
479
480struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
481 netdev_features_t features, bool is_ipv6)
482{
483 struct sock *sk = gso_skb->sk;
484 unsigned int sum_truesize = 0;
485 struct sk_buff *segs, *seg;
486 struct udphdr *uh;
487 unsigned int mss;
488 bool copy_dtor;
489 __sum16 check;
490 __be16 newlen;
491 int ret = 0;
492
493 mss = skb_shinfo(gso_skb)->gso_size;
494 if (gso_skb->len <= sizeof(*uh) + mss)
495 return ERR_PTR(error: -EINVAL);
496
497 if (unlikely(skb_checksum_start(gso_skb) !=
498 skb_transport_header(gso_skb) &&
499 !(skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST)))
500 return ERR_PTR(error: -EINVAL);
501
502 /* We don't know if egress device can segment and checksum the packet
503 * when IPv6 extension headers are present. Fall back to software GSO.
504 */
505 if (gso_skb->ip_summed != CHECKSUM_PARTIAL)
506 features &= ~(NETIF_F_GSO_UDP_L4 | NETIF_F_CSUM_MASK);
507
508 if (skb_gso_ok(skb: gso_skb, features: features | NETIF_F_GSO_ROBUST)) {
509 /* Packet is from an untrusted source, reset gso_segs. */
510 skb_shinfo(gso_skb)->gso_segs = DIV_ROUND_UP(gso_skb->len - sizeof(*uh),
511 mss);
512 return NULL;
513 }
514
515 if (skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST) {
516 /* Detect modified geometry and pass those to skb_segment. */
517 if ((skb_pagelen(skb: gso_skb) - sizeof(*uh) == skb_shinfo(gso_skb)->gso_size) &&
518 !(skb_shinfo(gso_skb)->gso_type & SKB_GSO_DODGY))
519 return __udp_gso_segment_list(skb: gso_skb, features, is_ipv6);
520
521 ret = __skb_linearize(skb: gso_skb);
522 if (ret)
523 return ERR_PTR(error: ret);
524
525 /* Setup csum, as fraglist skips this in udp4_gro_receive. */
526 gso_skb->csum_start = skb_transport_header(skb: gso_skb) - gso_skb->head;
527 gso_skb->csum_offset = offsetof(struct udphdr, check);
528 gso_skb->ip_summed = CHECKSUM_PARTIAL;
529
530 uh = udp_hdr(skb: gso_skb);
531 if (is_ipv6)
532 uh->check = ~udp_v6_check(len: gso_skb->len,
533 saddr: &ipv6_hdr(skb: gso_skb)->saddr,
534 daddr: &ipv6_hdr(skb: gso_skb)->daddr, base: 0);
535 else
536 uh->check = ~udp_v4_check(len: gso_skb->len,
537 saddr: ip_hdr(skb: gso_skb)->saddr,
538 daddr: ip_hdr(skb: gso_skb)->daddr, base: 0);
539 }
540
541 skb_pull(skb: gso_skb, len: sizeof(*uh));
542
543 /* clear destructor to avoid skb_segment assigning it to tail */
544 copy_dtor = gso_skb->destructor == sock_wfree;
545 if (copy_dtor) {
546 gso_skb->destructor = NULL;
547 gso_skb->sk = NULL;
548 }
549
550 segs = skb_segment(skb: gso_skb, features);
551 if (IS_ERR_OR_NULL(ptr: segs)) {
552 if (copy_dtor) {
553 gso_skb->destructor = sock_wfree;
554 gso_skb->sk = sk;
555 }
556 return segs;
557 }
558
559 /* GSO partial and frag_list segmentation only requires splitting
560 * the frame into an MSS multiple and possibly a remainder, both
561 * cases return a GSO skb. So update the mss now.
562 */
563 if (skb_is_gso(skb: segs))
564 mss *= skb_shinfo(segs)->gso_segs;
565
566 seg = segs;
567 uh = udp_hdr(skb: seg);
568
569 /* preserve TX timestamp flags and TS key for first segment */
570 skb_shinfo(seg)->tskey = skb_shinfo(gso_skb)->tskey;
571 skb_shinfo(seg)->tx_flags |=
572 (skb_shinfo(gso_skb)->tx_flags & SKBTX_ANY_TSTAMP);
573
574 /* compute checksum adjustment based on old length versus new */
575 newlen = htons(sizeof(*uh) + mss);
576 check = csum16_add(csum: csum16_sub(csum: uh->check, addend: uh->len), addend: newlen);
577
578 for (;;) {
579 if (copy_dtor) {
580 seg->destructor = sock_wfree;
581 seg->sk = sk;
582 sum_truesize += seg->truesize;
583 }
584
585 if (!seg->next)
586 break;
587
588 uh->len = newlen;
589 uh->check = check;
590
591 if (seg->ip_summed == CHECKSUM_PARTIAL)
592 gso_reset_checksum(skb: seg, res: ~check);
593 else
594 uh->check = gso_make_checksum(skb: seg, res: ~check) ? :
595 CSUM_MANGLED_0;
596
597 seg = seg->next;
598 uh = udp_hdr(skb: seg);
599 }
600
601 /* last packet can be partial gso_size, account for that in checksum */
602 newlen = htons(skb_tail_pointer(seg) - skb_transport_header(seg) +
603 seg->data_len);
604 check = csum16_add(csum: csum16_sub(csum: uh->check, addend: uh->len), addend: newlen);
605
606 uh->len = newlen;
607 uh->check = check;
608
609 if (seg->ip_summed == CHECKSUM_PARTIAL)
610 gso_reset_checksum(skb: seg, res: ~check);
611 else
612 uh->check = gso_make_checksum(skb: seg, res: ~check) ? : CSUM_MANGLED_0;
613
614 /* On the TX path, CHECKSUM_NONE and CHECKSUM_UNNECESSARY have the same
615 * meaning. However, check for bad offloads in the GSO stack expects the
616 * latter, if the checksum was calculated in software. To vouch for the
617 * segment skbs we actually need to set it on the gso_skb.
618 */
619 if (gso_skb->ip_summed == CHECKSUM_NONE)
620 gso_skb->ip_summed = CHECKSUM_UNNECESSARY;
621
622 /* update refcount for the packet */
623 if (copy_dtor) {
624 int delta = sum_truesize - gso_skb->truesize;
625
626 /* In some pathological cases, delta can be negative.
627 * We need to either use refcount_add() or refcount_sub_and_test()
628 */
629 if (likely(delta >= 0))
630 refcount_add(i: delta, r: &sk->sk_wmem_alloc);
631 else
632 WARN_ON_ONCE(refcount_sub_and_test(-delta, &sk->sk_wmem_alloc));
633 }
634 return segs;
635}
636EXPORT_SYMBOL_GPL(__udp_gso_segment);
637
638static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
639 netdev_features_t features)
640{
641 struct sk_buff *segs = ERR_PTR(error: -EINVAL);
642 unsigned int mss;
643 __wsum csum;
644 struct udphdr *uh;
645 struct iphdr *iph;
646
647 if (skb->encapsulation &&
648 (skb_shinfo(skb)->gso_type &
649 (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) {
650 segs = skb_udp_tunnel_segment(skb, features, false);
651 goto out;
652 }
653
654 if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))
655 goto out;
656
657 if (!pskb_may_pull(skb, len: sizeof(struct udphdr)))
658 goto out;
659
660 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
661 return __udp_gso_segment(skb, features, false);
662
663 mss = skb_shinfo(skb)->gso_size;
664 if (unlikely(skb->len <= mss))
665 goto out;
666
667 /* Do software UFO. Complete and fill in the UDP checksum as
668 * HW cannot do checksum of UDP packets sent as multiple
669 * IP fragments.
670 */
671
672 uh = udp_hdr(skb);
673 iph = ip_hdr(skb);
674
675 uh->check = 0;
676 csum = skb_checksum(skb, offset: 0, len: skb->len, csum: 0);
677 uh->check = udp_v4_check(len: skb->len, saddr: iph->saddr, daddr: iph->daddr, base: csum);
678 if (uh->check == 0)
679 uh->check = CSUM_MANGLED_0;
680
681 skb->ip_summed = CHECKSUM_UNNECESSARY;
682
683 /* If there is no outer header we can fake a checksum offload
684 * due to the fact that we have already done the checksum in
685 * software prior to segmenting the frame.
686 */
687 if (!skb->encap_hdr_csum)
688 features |= NETIF_F_HW_CSUM;
689
690 /* Fragment the skb. IP headers of the fragments are updated in
691 * inet_gso_segment()
692 */
693 segs = skb_segment(skb, features);
694out:
695 return segs;
696}
697
698
699#define UDP_GRO_CNT_MAX 64
700static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
701 struct sk_buff *skb)
702{
703 struct udphdr *uh = udp_gro_udphdr(skb);
704 struct sk_buff *pp = NULL;
705 struct udphdr *uh2;
706 struct sk_buff *p;
707 unsigned int ulen;
708 int ret = 0;
709 int flush;
710
711 /* requires non zero csum, for symmetry with GSO */
712 if (!uh->check) {
713 NAPI_GRO_CB(skb)->flush = 1;
714 return NULL;
715 }
716
717 /* Do not deal with padded or malicious packets, sorry ! */
718 ulen = ntohs(uh->len);
719 if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) {
720 NAPI_GRO_CB(skb)->flush = 1;
721 return NULL;
722 }
723 /* pull encapsulating udp header */
724 skb_gro_pull(skb, len: sizeof(struct udphdr));
725
726 list_for_each_entry(p, head, list) {
727 if (!NAPI_GRO_CB(p)->same_flow)
728 continue;
729
730 uh2 = udp_hdr(skb: p);
731
732 /* Match ports only, as csum is always non zero */
733 if ((*(u32 *)&uh->source != *(u32 *)&uh2->source)) {
734 NAPI_GRO_CB(p)->same_flow = 0;
735 continue;
736 }
737
738 if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {
739 NAPI_GRO_CB(skb)->flush = 1;
740 return p;
741 }
742
743 flush = gro_receive_network_flush(th: uh, th2: uh2, p);
744
745 /* Terminate the flow on len mismatch or if it grow "too much".
746 * Under small packet flood GRO count could elsewhere grow a lot
747 * leading to excessive truesize values.
748 * On len mismatch merge the first packet shorter than gso_size,
749 * otherwise complete the GRO packet.
750 */
751 if (ulen > ntohs(uh2->len) || flush) {
752 pp = p;
753 } else {
754 if (NAPI_GRO_CB(skb)->is_flist) {
755 if (!pskb_may_pull(skb, len: skb_gro_offset(skb))) {
756 NAPI_GRO_CB(skb)->flush = 1;
757 return NULL;
758 }
759 if ((skb->ip_summed != p->ip_summed) ||
760 (skb->csum_level != p->csum_level)) {
761 NAPI_GRO_CB(skb)->flush = 1;
762 return NULL;
763 }
764 skb_set_network_header(skb, offset: skb_gro_receive_network_offset(skb));
765 ret = skb_gro_receive_list(p, skb);
766 } else {
767 skb_gro_postpull_rcsum(skb, start: uh,
768 len: sizeof(struct udphdr));
769
770 ret = skb_gro_receive(p, skb);
771 }
772 }
773
774 if (ret || ulen != ntohs(uh2->len) ||
775 NAPI_GRO_CB(p)->count >= UDP_GRO_CNT_MAX)
776 pp = p;
777
778 return pp;
779 }
780
781 /* mismatch, but we never need to flush */
782 return NULL;
783}
784
785struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
786 struct udphdr *uh, struct sock *sk)
787{
788 struct sk_buff *pp = NULL;
789 struct sk_buff *p;
790 struct udphdr *uh2;
791 unsigned int off = skb_gro_offset(skb);
792 int flush = 1;
793
794 /* We can do L4 aggregation only if the packet can't land in a tunnel
795 * otherwise we could corrupt the inner stream. Detecting such packets
796 * cannot be foolproof and the aggregation might still happen in some
797 * cases. Such packets should be caught in udp_unexpected_gso later.
798 */
799 NAPI_GRO_CB(skb)->is_flist = 0;
800 if (!sk || !udp_sk(sk)->gro_receive) {
801 /* If the packet was locally encapsulated in a UDP tunnel that
802 * wasn't detected above, do not GRO.
803 */
804 if (skb->encapsulation)
805 goto out;
806
807 if (skb->dev->features & NETIF_F_GRO_FRAGLIST)
808 NAPI_GRO_CB(skb)->is_flist = sk ? !udp_test_bit(GRO_ENABLED, sk) : 1;
809
810 if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) ||
811 (sk && udp_test_bit(GRO_ENABLED, sk)) || NAPI_GRO_CB(skb)->is_flist)
812 return call_gro_receive(cb: udp_gro_receive_segment, head, skb);
813
814 /* no GRO, be sure flush the current packet */
815 goto out;
816 }
817
818 if (NAPI_GRO_CB(skb)->encap_mark ||
819 (uh->check && skb->ip_summed != CHECKSUM_PARTIAL &&
820 NAPI_GRO_CB(skb)->csum_cnt == 0 &&
821 !NAPI_GRO_CB(skb)->csum_valid))
822 goto out;
823
824 /* mark that this skb passed once through the tunnel gro layer */
825 NAPI_GRO_CB(skb)->encap_mark = 1;
826
827 flush = 0;
828
829 list_for_each_entry(p, head, list) {
830 if (!NAPI_GRO_CB(p)->same_flow)
831 continue;
832
833 uh2 = (struct udphdr *)(p->data + off);
834
835 /* Match ports and either checksums are either both zero
836 * or nonzero.
837 */
838 if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) ||
839 (!uh->check ^ !uh2->check)) {
840 NAPI_GRO_CB(p)->same_flow = 0;
841 continue;
842 }
843 }
844
845 skb_gro_pull(skb, len: sizeof(struct udphdr)); /* pull encapsulating udp header */
846 skb_gro_postpull_rcsum(skb, start: uh, len: sizeof(struct udphdr));
847 pp = udp_tunnel_gro_rcv(sk, head, skb);
848
849out:
850 skb_gro_flush_final(skb, pp, flush);
851 return pp;
852}
853EXPORT_SYMBOL(udp_gro_receive);
854
855static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
856 __be16 dport)
857{
858 const struct iphdr *iph = skb_gro_network_header(skb);
859 struct net *net = dev_net_rcu(dev: skb->dev);
860 struct sock *sk;
861 int iif, sdif;
862
863 sk = udp_tunnel_sk(net, is_ipv6: false);
864 if (sk && dport == htons(sk->sk_num))
865 return sk;
866
867 inet_get_iif_sdif(skb, iif: &iif, sdif: &sdif);
868
869 return __udp4_lib_lookup(net, saddr: iph->saddr, sport,
870 daddr: iph->daddr, dport, dif: iif,
871 sdif, tbl: net->ipv4.udp_table, NULL);
872}
873
874INDIRECT_CALLABLE_SCOPE
875struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)
876{
877 struct udphdr *uh = udp_gro_udphdr(skb);
878 struct sock *sk = NULL;
879 struct sk_buff *pp;
880
881 if (unlikely(!uh))
882 goto flush;
883
884 /* Don't bother verifying checksum if we're going to flush anyway. */
885 if (NAPI_GRO_CB(skb)->flush)
886 goto skip;
887
888 if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
889 inet_gro_compute_pseudo))
890 goto flush;
891 else if (uh->check)
892 skb_gro_checksum_try_convert(skb, IPPROTO_UDP,
893 inet_gro_compute_pseudo);
894skip:
895 if (static_branch_unlikely(&udp_encap_needed_key))
896 sk = udp4_gro_lookup_skb(skb, sport: uh->source, dport: uh->dest);
897
898 pp = udp_gro_receive(head, skb, uh, sk);
899 return pp;
900
901flush:
902 NAPI_GRO_CB(skb)->flush = 1;
903 return NULL;
904}
905
906static int udp_gro_complete_segment(struct sk_buff *skb)
907{
908 struct udphdr *uh = udp_hdr(skb);
909
910 skb->csum_start = (unsigned char *)uh - skb->head;
911 skb->csum_offset = offsetof(struct udphdr, check);
912 skb->ip_summed = CHECKSUM_PARTIAL;
913
914 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
915 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;
916
917 if (skb->encapsulation)
918 skb->inner_transport_header = skb->transport_header;
919
920 return 0;
921}
922
923int udp_gro_complete(struct sk_buff *skb, int nhoff,
924 udp_lookup_t lookup)
925{
926 __be16 newlen = htons(skb->len - nhoff);
927 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
928 struct sock *sk;
929 int err;
930
931 uh->len = newlen;
932
933 sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb,
934 udp4_lib_lookup_skb, skb, uh->source, uh->dest);
935 if (sk && udp_sk(sk)->gro_complete) {
936 skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM
937 : SKB_GSO_UDP_TUNNEL;
938
939 /* clear the encap mark, so that inner frag_list gro_complete
940 * can take place
941 */
942 NAPI_GRO_CB(skb)->encap_mark = 0;
943
944 /* Set encapsulation before calling into inner gro_complete()
945 * functions to make them set up the inner offsets.
946 */
947 skb->encapsulation = 1;
948 err = udp_sk(sk)->gro_complete(sk, skb,
949 nhoff + sizeof(struct udphdr));
950 } else {
951 err = udp_gro_complete_segment(skb);
952 }
953
954 if (skb->remcsum_offload)
955 skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;
956
957 return err;
958}
959EXPORT_SYMBOL(udp_gro_complete);
960
961INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)
962{
963 const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
964 const struct iphdr *iph = (struct iphdr *)(skb->data + offset);
965 struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
966
967 /* do fraglist only if there is no outer UDP encap (or we already processed it) */
968 if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {
969 uh->len = htons(skb->len - nhoff);
970
971 skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
972 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
973
974 __skb_incr_checksum_unnecessary(skb);
975
976 return 0;
977 }
978
979 if (uh->check)
980 uh->check = ~udp_v4_check(len: skb->len - nhoff, saddr: iph->saddr,
981 daddr: iph->daddr, base: 0);
982
983 return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
984}
985
986int __init udpv4_offload_init(void)
987{
988 net_hotdata.udpv4_offload = (struct net_offload) {
989 .callbacks = {
990 .gso_segment = udp4_ufo_fragment,
991 .gro_receive = udp4_gro_receive,
992 .gro_complete = udp4_gro_complete,
993 },
994 };
995
996 return inet_add_offload(prot: &net_hotdata.udpv4_offload, IPPROTO_UDP);
997}
998

source code of linux/net/ipv4/udp_offload.c