1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * IPV6 GSO/GRO offload support
4 * Linux INET implementation
5 *
6 * Copyright (C) 2016 secunet Security Networks AG
7 * Author: Steffen Klassert <steffen.klassert@secunet.com>
8 *
9 * ESP GRO support
10 */
11
12#include <linux/skbuff.h>
13#include <linux/init.h>
14#include <net/protocol.h>
15#include <crypto/aead.h>
16#include <crypto/authenc.h>
17#include <linux/err.h>
18#include <linux/module.h>
19#include <net/gro.h>
20#include <net/gso.h>
21#include <net/ip.h>
22#include <net/xfrm.h>
23#include <net/esp.h>
24#include <linux/scatterlist.h>
25#include <linux/kernel.h>
26#include <linux/slab.h>
27#include <linux/spinlock.h>
28#include <net/ip6_route.h>
29#include <net/ipv6.h>
30#include <linux/icmpv6.h>
31
32static __u16 esp6_nexthdr_esp_offset(struct ipv6hdr *ipv6_hdr, int nhlen)
33{
34 int off = sizeof(struct ipv6hdr);
35 struct ipv6_opt_hdr *exthdr;
36
37 /* ESP or ESPINUDP */
38 if (likely(ipv6_hdr->nexthdr == NEXTHDR_ESP ||
39 ipv6_hdr->nexthdr == NEXTHDR_UDP))
40 return offsetof(struct ipv6hdr, nexthdr);
41
42 while (off < nhlen) {
43 exthdr = (void *)ipv6_hdr + off;
44 if (exthdr->nexthdr == NEXTHDR_ESP)
45 return off;
46
47 off += ipv6_optlen(exthdr);
48 }
49
50 return 0;
51}
52
53static struct sk_buff *esp6_gro_receive(struct list_head *head,
54 struct sk_buff *skb)
55{
56 int offset = skb_gro_offset(skb);
57 struct xfrm_offload *xo;
58 struct xfrm_state *x;
59 int encap_type = 0;
60 __be32 seq;
61 __be32 spi;
62 int nhoff;
63
64 if (NAPI_GRO_CB(skb)->proto == IPPROTO_UDP)
65 encap_type = UDP_ENCAP_ESPINUDP;
66
67 if (!pskb_pull(skb, len: offset))
68 return NULL;
69
70 if (xfrm_parse_spi(skb, IPPROTO_ESP, spi: &spi, seq: &seq) != 0)
71 goto out;
72
73 xo = xfrm_offload(skb);
74 if (!xo || !(xo->flags & CRYPTO_DONE)) {
75 struct sec_path *sp = secpath_set(skb);
76
77 if (!sp)
78 goto out;
79
80 if (sp->len == XFRM_MAX_DEPTH)
81 goto out_reset;
82
83 x = xfrm_input_state_lookup(net: dev_net(dev: skb->dev), mark: skb->mark,
84 daddr: (xfrm_address_t *)&ipv6_hdr(skb)->daddr,
85 spi, IPPROTO_ESP, AF_INET6);
86
87 if (unlikely(x && x->dir && x->dir != XFRM_SA_DIR_IN)) {
88 /* non-offload path will record the error and audit log */
89 xfrm_state_put(x);
90 x = NULL;
91 }
92
93 if (!x)
94 goto out_reset;
95
96 skb->mark = xfrm_smark_get(mark: skb->mark, x);
97
98 sp->xvec[sp->len++] = x;
99 sp->olen++;
100
101 xo = xfrm_offload(skb);
102 if (!xo)
103 goto out_reset;
104 }
105
106 xo->flags |= XFRM_GRO;
107
108 nhoff = esp6_nexthdr_esp_offset(ipv6_hdr: ipv6_hdr(skb), nhlen: offset);
109 if (!nhoff)
110 goto out;
111
112 IP6CB(skb)->nhoff = nhoff;
113 XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
114 XFRM_SPI_SKB_CB(skb)->family = AF_INET6;
115 XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);
116 XFRM_SPI_SKB_CB(skb)->seq = seq;
117
118 /* We don't need to handle errors from xfrm_input, it does all
119 * the error handling and frees the resources on error. */
120 xfrm_input(skb, IPPROTO_ESP, spi, encap_type);
121
122 return ERR_PTR(error: -EINPROGRESS);
123out_reset:
124 secpath_reset(skb);
125out:
126 skb_push(skb, len: offset);
127 NAPI_GRO_CB(skb)->same_flow = 0;
128 NAPI_GRO_CB(skb)->flush = 1;
129
130 return NULL;
131}
132
133static void esp6_gso_encap(struct xfrm_state *x, struct sk_buff *skb)
134{
135 struct ip_esp_hdr *esph;
136 struct ipv6hdr *iph = ipv6_hdr(skb);
137 struct xfrm_offload *xo = xfrm_offload(skb);
138 u8 proto = iph->nexthdr;
139
140 skb_push(skb, len: -skb_network_offset(skb));
141
142 if (x->outer_mode.encap == XFRM_MODE_TRANSPORT) {
143 __be16 frag;
144
145 ipv6_skip_exthdr(skb, start: sizeof(struct ipv6hdr), nexthdrp: &proto, frag_offp: &frag);
146 }
147
148 esph = ip_esp_hdr(skb);
149 *skb_mac_header(skb) = IPPROTO_ESP;
150
151 esph->spi = x->id.spi;
152 esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
153
154 xo->proto = proto;
155}
156
157static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x,
158 struct sk_buff *skb,
159 netdev_features_t features)
160{
161 struct xfrm_offload *xo = xfrm_offload(skb);
162 const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, ipproto: xo->proto);
163 __be16 type = inner_mode->family == AF_INET ? htons(ETH_P_IP)
164 : htons(ETH_P_IPV6);
165
166 return skb_eth_gso_segment(skb, features, type);
167}
168
169static struct sk_buff *xfrm6_transport_gso_segment(struct xfrm_state *x,
170 struct sk_buff *skb,
171 netdev_features_t features)
172{
173 const struct net_offload *ops;
174 struct sk_buff *segs = ERR_PTR(error: -EINVAL);
175 struct xfrm_offload *xo = xfrm_offload(skb);
176
177 skb->transport_header += x->props.header_len;
178 ops = rcu_dereference(inet6_offloads[xo->proto]);
179 if (likely(ops && ops->callbacks.gso_segment))
180 segs = ops->callbacks.gso_segment(skb, features);
181
182 return segs;
183}
184
185static struct sk_buff *xfrm6_beet_gso_segment(struct xfrm_state *x,
186 struct sk_buff *skb,
187 netdev_features_t features)
188{
189 struct xfrm_offload *xo = xfrm_offload(skb);
190 struct sk_buff *segs = ERR_PTR(error: -EINVAL);
191 const struct net_offload *ops;
192 u8 proto = xo->proto;
193
194 skb->transport_header += x->props.header_len;
195
196 if (x->sel.family != AF_INET6) {
197 skb->transport_header -=
198 (sizeof(struct ipv6hdr) - sizeof(struct iphdr));
199
200 if (proto == IPPROTO_BEETPH) {
201 struct ip_beet_phdr *ph =
202 (struct ip_beet_phdr *)skb->data;
203
204 skb->transport_header += ph->hdrlen * 8;
205 proto = ph->nexthdr;
206 } else {
207 skb->transport_header -= IPV4_BEET_PHMAXLEN;
208 }
209
210 if (proto == IPPROTO_TCP)
211 skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
212 } else {
213 __be16 frag;
214
215 skb->transport_header +=
216 ipv6_skip_exthdr(skb, start: 0, nexthdrp: &proto, frag_offp: &frag);
217 }
218
219 if (proto == IPPROTO_IPIP)
220 skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
221
222 __skb_pull(skb, len: skb_transport_offset(skb));
223 ops = rcu_dereference(inet6_offloads[proto]);
224 if (likely(ops && ops->callbacks.gso_segment))
225 segs = ops->callbacks.gso_segment(skb, features);
226
227 return segs;
228}
229
230static struct sk_buff *xfrm6_outer_mode_gso_segment(struct xfrm_state *x,
231 struct sk_buff *skb,
232 netdev_features_t features)
233{
234 switch (x->outer_mode.encap) {
235 case XFRM_MODE_TUNNEL:
236 return xfrm6_tunnel_gso_segment(x, skb, features);
237 case XFRM_MODE_TRANSPORT:
238 return xfrm6_transport_gso_segment(x, skb, features);
239 case XFRM_MODE_BEET:
240 return xfrm6_beet_gso_segment(x, skb, features);
241 }
242
243 return ERR_PTR(error: -EOPNOTSUPP);
244}
245
246static struct sk_buff *esp6_gso_segment(struct sk_buff *skb,
247 netdev_features_t features)
248{
249 struct xfrm_state *x;
250 struct ip_esp_hdr *esph;
251 struct crypto_aead *aead;
252 netdev_features_t esp_features = features;
253 struct xfrm_offload *xo = xfrm_offload(skb);
254 struct sec_path *sp;
255
256 if (!xo)
257 return ERR_PTR(error: -EINVAL);
258
259 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_ESP))
260 return ERR_PTR(error: -EINVAL);
261
262 sp = skb_sec_path(skb);
263 x = sp->xvec[sp->len - 1];
264 aead = x->data;
265 esph = ip_esp_hdr(skb);
266
267 if (esph->spi != x->id.spi)
268 return ERR_PTR(error: -EINVAL);
269
270 if (!pskb_may_pull(skb, len: sizeof(*esph) + crypto_aead_ivsize(tfm: aead)))
271 return ERR_PTR(error: -EINVAL);
272
273 __skb_pull(skb, len: sizeof(*esph) + crypto_aead_ivsize(tfm: aead));
274
275 skb->encap_hdr_csum = 1;
276
277 if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev)
278 esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK |
279 NETIF_F_SCTP_CRC);
280 else if (!(features & NETIF_F_HW_ESP_TX_CSUM))
281 esp_features = features & ~(NETIF_F_CSUM_MASK |
282 NETIF_F_SCTP_CRC);
283
284 xo->flags |= XFRM_GSO_SEGMENT;
285
286 return xfrm6_outer_mode_gso_segment(x, skb, features: esp_features);
287}
288
289static int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb)
290{
291 struct crypto_aead *aead = x->data;
292 struct xfrm_offload *xo = xfrm_offload(skb);
293
294 if (!pskb_may_pull(skb, len: sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(tfm: aead)))
295 return -EINVAL;
296
297 if (!(xo->flags & CRYPTO_DONE))
298 skb->ip_summed = CHECKSUM_NONE;
299
300 return esp6_input_done2(skb, err: 0);
301}
302
303static int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features)
304{
305 int len;
306 int err;
307 int alen;
308 int blksize;
309 struct xfrm_offload *xo;
310 struct crypto_aead *aead;
311 struct esp_info esp;
312 bool hw_offload = true;
313 __u32 seq;
314
315 esp.inplace = true;
316
317 xo = xfrm_offload(skb);
318
319 if (!xo)
320 return -EINVAL;
321
322 if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev) {
323 xo->flags |= CRYPTO_FALLBACK;
324 hw_offload = false;
325 }
326
327 esp.proto = xo->proto;
328
329 /* skb is pure payload to encrypt */
330
331 aead = x->data;
332 alen = crypto_aead_authsize(tfm: aead);
333
334 esp.tfclen = 0;
335 /* XXX: Add support for tfc padding here. */
336
337 blksize = ALIGN(crypto_aead_blocksize(aead), 4);
338 esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
339 esp.plen = esp.clen - skb->len - esp.tfclen;
340 esp.tailen = esp.tfclen + esp.plen + alen;
341
342 if (!hw_offload || !skb_is_gso(skb)) {
343 esp.nfrags = esp6_output_head(x, skb, esp: &esp);
344 if (esp.nfrags < 0)
345 return esp.nfrags;
346 }
347
348 seq = xo->seq.low;
349
350 esp.esph = ip_esp_hdr(skb);
351 esp.esph->spi = x->id.spi;
352
353 skb_push(skb, len: -skb_network_offset(skb));
354
355 if (xo->flags & XFRM_GSO_SEGMENT) {
356 esp.esph->seq_no = htonl(seq);
357
358 if (!skb_is_gso(skb))
359 xo->seq.low++;
360 else
361 xo->seq.low += skb_shinfo(skb)->gso_segs;
362 }
363
364 if (xo->seq.low < seq)
365 xo->seq.hi++;
366
367 esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));
368
369 len = skb->len - sizeof(struct ipv6hdr);
370 if (len > IPV6_MAXPLEN)
371 len = 0;
372
373 ipv6_hdr(skb)->payload_len = htons(len);
374
375 if (hw_offload) {
376 if (!skb_ext_add(skb, id: SKB_EXT_SEC_PATH))
377 return -ENOMEM;
378
379 xo = xfrm_offload(skb);
380 if (!xo)
381 return -EINVAL;
382
383 xo->flags |= XFRM_XMIT;
384 return 0;
385 }
386
387 err = esp6_output_tail(x, skb, esp: &esp);
388 if (err)
389 return err;
390
391 secpath_reset(skb);
392
393 if (skb_needs_linearize(skb, features: skb->dev->features) &&
394 __skb_linearize(skb))
395 return -ENOMEM;
396 return 0;
397}
398
399static const struct net_offload esp6_offload = {
400 .callbacks = {
401 .gro_receive = esp6_gro_receive,
402 .gso_segment = esp6_gso_segment,
403 },
404};
405
406static const struct xfrm_type_offload esp6_type_offload = {
407 .owner = THIS_MODULE,
408 .proto = IPPROTO_ESP,
409 .input_tail = esp6_input_tail,
410 .xmit = esp6_xmit,
411 .encap = esp6_gso_encap,
412};
413
414static int __init esp6_offload_init(void)
415{
416 if (xfrm_register_type_offload(type: &esp6_type_offload, AF_INET6) < 0) {
417 pr_info("%s: can't add xfrm type offload\n", __func__);
418 return -EAGAIN;
419 }
420
421 return inet6_add_offload(prot: &esp6_offload, IPPROTO_ESP);
422}
423
424static void __exit esp6_offload_exit(void)
425{
426 xfrm_unregister_type_offload(type: &esp6_type_offload, AF_INET6);
427 inet6_del_offload(prot: &esp6_offload, IPPROTO_ESP);
428}
429
430module_init(esp6_offload_init);
431module_exit(esp6_offload_exit);
432MODULE_LICENSE("GPL");
433MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
434MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET6, XFRM_PROTO_ESP);
435MODULE_DESCRIPTION("IPV6 GSO/GRO offload support");
436

source code of linux/net/ipv6/esp6_offload.c