1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * xfrm_input.c |
4 | * |
5 | * Changes: |
6 | * YOSHIFUJI Hideaki @USAGI |
7 | * Split up af-specific portion |
8 | * |
9 | */ |
10 | |
11 | #include <linux/bottom_half.h> |
12 | #include <linux/cache.h> |
13 | #include <linux/interrupt.h> |
14 | #include <linux/slab.h> |
15 | #include <linux/module.h> |
16 | #include <linux/netdevice.h> |
17 | #include <linux/percpu.h> |
18 | #include <net/dst.h> |
19 | #include <net/ip.h> |
20 | #include <net/xfrm.h> |
21 | #include <net/ip_tunnels.h> |
22 | #include <net/ip6_tunnel.h> |
23 | #include <net/dst_metadata.h> |
24 | #include <net/hotdata.h> |
25 | |
26 | #include "xfrm_inout.h" |
27 | |
28 | struct xfrm_trans_tasklet { |
29 | struct work_struct work; |
30 | spinlock_t queue_lock; |
31 | struct sk_buff_head queue; |
32 | }; |
33 | |
34 | struct xfrm_trans_cb { |
35 | union { |
36 | struct inet_skb_parm h4; |
37 | #if IS_ENABLED(CONFIG_IPV6) |
38 | struct inet6_skb_parm h6; |
39 | #endif |
40 | } ; |
41 | int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb); |
42 | struct net *net; |
43 | }; |
44 | |
45 | #define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0])) |
46 | |
47 | static DEFINE_SPINLOCK(xfrm_input_afinfo_lock); |
48 | static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[2][AF_INET6 + 1]; |
49 | |
50 | static struct gro_cells gro_cells; |
51 | static struct net_device xfrm_napi_dev; |
52 | |
53 | static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet); |
54 | |
55 | int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo) |
56 | { |
57 | int err = 0; |
58 | |
59 | if (WARN_ON(afinfo->family > AF_INET6)) |
60 | return -EAFNOSUPPORT; |
61 | |
62 | spin_lock_bh(lock: &xfrm_input_afinfo_lock); |
63 | if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family])) |
64 | err = -EEXIST; |
65 | else |
66 | rcu_assign_pointer(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], afinfo); |
67 | spin_unlock_bh(lock: &xfrm_input_afinfo_lock); |
68 | return err; |
69 | } |
70 | EXPORT_SYMBOL(xfrm_input_register_afinfo); |
71 | |
72 | int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo) |
73 | { |
74 | int err = 0; |
75 | |
76 | spin_lock_bh(lock: &xfrm_input_afinfo_lock); |
77 | if (likely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family])) { |
78 | if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family] != afinfo)) |
79 | err = -EINVAL; |
80 | else |
81 | RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], NULL); |
82 | } |
83 | spin_unlock_bh(lock: &xfrm_input_afinfo_lock); |
84 | synchronize_rcu(); |
85 | return err; |
86 | } |
87 | EXPORT_SYMBOL(xfrm_input_unregister_afinfo); |
88 | |
89 | static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(u8 family, bool is_ipip) |
90 | { |
91 | const struct xfrm_input_afinfo *afinfo; |
92 | |
93 | if (WARN_ON_ONCE(family > AF_INET6)) |
94 | return NULL; |
95 | |
96 | rcu_read_lock(); |
97 | afinfo = rcu_dereference(xfrm_input_afinfo[is_ipip][family]); |
98 | if (unlikely(!afinfo)) |
99 | rcu_read_unlock(); |
100 | return afinfo; |
101 | } |
102 | |
103 | static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol, |
104 | int err) |
105 | { |
106 | bool is_ipip = (protocol == IPPROTO_IPIP || protocol == IPPROTO_IPV6); |
107 | const struct xfrm_input_afinfo *afinfo; |
108 | int ret; |
109 | |
110 | afinfo = xfrm_input_get_afinfo(family, is_ipip); |
111 | if (!afinfo) |
112 | return -EAFNOSUPPORT; |
113 | |
114 | ret = afinfo->callback(skb, protocol, err); |
115 | rcu_read_unlock(); |
116 | |
117 | return ret; |
118 | } |
119 | |
120 | struct sec_path *secpath_set(struct sk_buff *skb) |
121 | { |
122 | struct sec_path *sp, *tmp = skb_ext_find(skb, id: SKB_EXT_SEC_PATH); |
123 | |
124 | sp = skb_ext_add(skb, id: SKB_EXT_SEC_PATH); |
125 | if (!sp) |
126 | return NULL; |
127 | |
128 | if (tmp) /* reused existing one (was COW'd if needed) */ |
129 | return sp; |
130 | |
131 | /* allocated new secpath */ |
132 | memset(sp->ovec, 0, sizeof(sp->ovec)); |
133 | sp->olen = 0; |
134 | sp->len = 0; |
135 | sp->verified_cnt = 0; |
136 | |
137 | return sp; |
138 | } |
139 | EXPORT_SYMBOL(secpath_set); |
140 | |
141 | /* Fetch spi and seq from ipsec header */ |
142 | |
143 | int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq) |
144 | { |
145 | int offset, offset_seq; |
146 | int hlen; |
147 | |
148 | switch (nexthdr) { |
149 | case IPPROTO_AH: |
150 | hlen = sizeof(struct ip_auth_hdr); |
151 | offset = offsetof(struct ip_auth_hdr, spi); |
152 | offset_seq = offsetof(struct ip_auth_hdr, seq_no); |
153 | break; |
154 | case IPPROTO_ESP: |
155 | hlen = sizeof(struct ip_esp_hdr); |
156 | offset = offsetof(struct ip_esp_hdr, spi); |
157 | offset_seq = offsetof(struct ip_esp_hdr, seq_no); |
158 | break; |
159 | case IPPROTO_COMP: |
160 | if (!pskb_may_pull(skb, len: sizeof(struct ip_comp_hdr))) |
161 | return -EINVAL; |
162 | *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2))); |
163 | *seq = 0; |
164 | return 0; |
165 | default: |
166 | return 1; |
167 | } |
168 | |
169 | if (!pskb_may_pull(skb, len: hlen)) |
170 | return -EINVAL; |
171 | |
172 | *spi = *(__be32 *)(skb_transport_header(skb) + offset); |
173 | *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq); |
174 | return 0; |
175 | } |
176 | EXPORT_SYMBOL(xfrm_parse_spi); |
177 | |
178 | static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb) |
179 | { |
180 | struct iphdr *iph; |
181 | int optlen = 0; |
182 | int err = -EINVAL; |
183 | |
184 | skb->protocol = htons(ETH_P_IP); |
185 | |
186 | if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) { |
187 | struct ip_beet_phdr *ph; |
188 | int phlen; |
189 | |
190 | if (!pskb_may_pull(skb, len: sizeof(*ph))) |
191 | goto out; |
192 | |
193 | ph = (struct ip_beet_phdr *)skb->data; |
194 | |
195 | phlen = sizeof(*ph) + ph->padlen; |
196 | optlen = ph->hdrlen * 8 + (IPV4_BEET_PHMAXLEN - phlen); |
197 | if (optlen < 0 || optlen & 3 || optlen > 250) |
198 | goto out; |
199 | |
200 | XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr; |
201 | |
202 | if (!pskb_may_pull(skb, len: phlen)) |
203 | goto out; |
204 | __skb_pull(skb, len: phlen); |
205 | } |
206 | |
207 | skb_push(skb, len: sizeof(*iph)); |
208 | skb_reset_network_header(skb); |
209 | skb_mac_header_rebuild(skb); |
210 | |
211 | xfrm4_beet_make_header(skb); |
212 | |
213 | iph = ip_hdr(skb); |
214 | |
215 | iph->ihl += optlen / 4; |
216 | iph->tot_len = htons(skb->len); |
217 | iph->daddr = x->sel.daddr.a4; |
218 | iph->saddr = x->sel.saddr.a4; |
219 | iph->check = 0; |
220 | iph->check = ip_fast_csum(iph: skb_network_header(skb), ihl: iph->ihl); |
221 | err = 0; |
222 | out: |
223 | return err; |
224 | } |
225 | |
226 | static void ipip_ecn_decapsulate(struct sk_buff *skb) |
227 | { |
228 | struct iphdr *inner_iph = ipip_hdr(skb); |
229 | |
230 | if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos)) |
231 | IP_ECN_set_ce(iph: inner_iph); |
232 | } |
233 | |
234 | static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb) |
235 | { |
236 | int err = -EINVAL; |
237 | |
238 | skb->protocol = htons(ETH_P_IP); |
239 | |
240 | if (!pskb_may_pull(skb, len: sizeof(struct iphdr))) |
241 | goto out; |
242 | |
243 | err = skb_unclone(skb, GFP_ATOMIC); |
244 | if (err) |
245 | goto out; |
246 | |
247 | if (x->props.flags & XFRM_STATE_DECAP_DSCP) |
248 | ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, inner: ipip_hdr(skb)); |
249 | if (!(x->props.flags & XFRM_STATE_NOECN)) |
250 | ipip_ecn_decapsulate(skb); |
251 | |
252 | skb_reset_network_header(skb); |
253 | skb_mac_header_rebuild(skb); |
254 | if (skb->mac_len) |
255 | eth_hdr(skb)->h_proto = skb->protocol; |
256 | |
257 | err = 0; |
258 | |
259 | out: |
260 | return err; |
261 | } |
262 | |
263 | static void ipip6_ecn_decapsulate(struct sk_buff *skb) |
264 | { |
265 | struct ipv6hdr *inner_iph = ipipv6_hdr(skb); |
266 | |
267 | if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos)) |
268 | IP6_ECN_set_ce(skb, iph: inner_iph); |
269 | } |
270 | |
271 | static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb) |
272 | { |
273 | int err = -EINVAL; |
274 | |
275 | skb->protocol = htons(ETH_P_IPV6); |
276 | |
277 | if (!pskb_may_pull(skb, len: sizeof(struct ipv6hdr))) |
278 | goto out; |
279 | |
280 | err = skb_unclone(skb, GFP_ATOMIC); |
281 | if (err) |
282 | goto out; |
283 | |
284 | if (x->props.flags & XFRM_STATE_DECAP_DSCP) |
285 | ipv6_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, inner: ipipv6_hdr(skb)); |
286 | if (!(x->props.flags & XFRM_STATE_NOECN)) |
287 | ipip6_ecn_decapsulate(skb); |
288 | |
289 | skb_reset_network_header(skb); |
290 | skb_mac_header_rebuild(skb); |
291 | if (skb->mac_len) |
292 | eth_hdr(skb)->h_proto = skb->protocol; |
293 | |
294 | err = 0; |
295 | |
296 | out: |
297 | return err; |
298 | } |
299 | |
300 | static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb) |
301 | { |
302 | struct ipv6hdr *ip6h; |
303 | int size = sizeof(struct ipv6hdr); |
304 | int err; |
305 | |
306 | skb->protocol = htons(ETH_P_IPV6); |
307 | |
308 | err = skb_cow_head(skb, headroom: size + skb->mac_len); |
309 | if (err) |
310 | goto out; |
311 | |
312 | __skb_push(skb, len: size); |
313 | skb_reset_network_header(skb); |
314 | skb_mac_header_rebuild(skb); |
315 | |
316 | xfrm6_beet_make_header(skb); |
317 | |
318 | ip6h = ipv6_hdr(skb); |
319 | ip6h->payload_len = htons(skb->len - size); |
320 | ip6h->daddr = x->sel.daddr.in6; |
321 | ip6h->saddr = x->sel.saddr.in6; |
322 | err = 0; |
323 | out: |
324 | return err; |
325 | } |
326 | |
327 | /* Remove encapsulation header. |
328 | * |
329 | * The IP header will be moved over the top of the encapsulation |
330 | * header. |
331 | * |
332 | * On entry, the transport header shall point to where the IP header |
333 | * should be and the network header shall be set to where the IP |
334 | * header currently is. skb->data shall point to the start of the |
335 | * payload. |
336 | */ |
337 | static int |
338 | xfrm_inner_mode_encap_remove(struct xfrm_state *x, |
339 | struct sk_buff *skb) |
340 | { |
341 | switch (x->props.mode) { |
342 | case XFRM_MODE_BEET: |
343 | switch (x->sel.family) { |
344 | case AF_INET: |
345 | return xfrm4_remove_beet_encap(x, skb); |
346 | case AF_INET6: |
347 | return xfrm6_remove_beet_encap(x, skb); |
348 | } |
349 | break; |
350 | case XFRM_MODE_TUNNEL: |
351 | switch (XFRM_MODE_SKB_CB(skb)->protocol) { |
352 | case IPPROTO_IPIP: |
353 | return xfrm4_remove_tunnel_encap(x, skb); |
354 | case IPPROTO_IPV6: |
355 | return xfrm6_remove_tunnel_encap(x, skb); |
356 | break; |
357 | } |
358 | return -EINVAL; |
359 | } |
360 | |
361 | WARN_ON_ONCE(1); |
362 | return -EOPNOTSUPP; |
363 | } |
364 | |
365 | static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb) |
366 | { |
367 | switch (x->props.family) { |
368 | case AF_INET: |
369 | xfrm4_extract_header(skb); |
370 | break; |
371 | case AF_INET6: |
372 | xfrm6_extract_header(skb); |
373 | break; |
374 | default: |
375 | WARN_ON_ONCE(1); |
376 | return -EAFNOSUPPORT; |
377 | } |
378 | |
379 | return xfrm_inner_mode_encap_remove(x, skb); |
380 | } |
381 | |
382 | /* Remove encapsulation header. |
383 | * |
384 | * The IP header will be moved over the top of the encapsulation header. |
385 | * |
386 | * On entry, skb_transport_header() shall point to where the IP header |
387 | * should be and skb_network_header() shall be set to where the IP header |
388 | * currently is. skb->data shall point to the start of the payload. |
389 | */ |
390 | static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb) |
391 | { |
392 | int ihl = skb->data - skb_transport_header(skb); |
393 | |
394 | if (skb->transport_header != skb->network_header) { |
395 | memmove(skb_transport_header(skb), |
396 | skb_network_header(skb), ihl); |
397 | skb->network_header = skb->transport_header; |
398 | } |
399 | ip_hdr(skb)->tot_len = htons(skb->len + ihl); |
400 | skb_reset_transport_header(skb); |
401 | return 0; |
402 | } |
403 | |
404 | static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb) |
405 | { |
406 | #if IS_ENABLED(CONFIG_IPV6) |
407 | int ihl = skb->data - skb_transport_header(skb); |
408 | |
409 | if (skb->transport_header != skb->network_header) { |
410 | memmove(skb_transport_header(skb), |
411 | skb_network_header(skb), ihl); |
412 | skb->network_header = skb->transport_header; |
413 | } |
414 | ipv6_hdr(skb)->payload_len = htons(skb->len + ihl - |
415 | sizeof(struct ipv6hdr)); |
416 | skb_reset_transport_header(skb); |
417 | return 0; |
418 | #else |
419 | WARN_ON_ONCE(1); |
420 | return -EAFNOSUPPORT; |
421 | #endif |
422 | } |
423 | |
424 | static int xfrm_inner_mode_input(struct xfrm_state *x, |
425 | struct sk_buff *skb) |
426 | { |
427 | switch (x->props.mode) { |
428 | case XFRM_MODE_BEET: |
429 | case XFRM_MODE_TUNNEL: |
430 | return xfrm_prepare_input(x, skb); |
431 | case XFRM_MODE_TRANSPORT: |
432 | if (x->props.family == AF_INET) |
433 | return xfrm4_transport_input(x, skb); |
434 | if (x->props.family == AF_INET6) |
435 | return xfrm6_transport_input(x, skb); |
436 | break; |
437 | case XFRM_MODE_ROUTEOPTIMIZATION: |
438 | WARN_ON_ONCE(1); |
439 | break; |
440 | default: |
441 | WARN_ON_ONCE(1); |
442 | break; |
443 | } |
444 | |
445 | return -EOPNOTSUPP; |
446 | } |
447 | |
448 | int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) |
449 | { |
450 | const struct xfrm_state_afinfo *afinfo; |
451 | struct net *net = dev_net(dev: skb->dev); |
452 | int err; |
453 | __be32 seq; |
454 | __be32 seq_hi; |
455 | struct xfrm_state *x = NULL; |
456 | xfrm_address_t *daddr; |
457 | u32 mark = skb->mark; |
458 | unsigned int family = AF_UNSPEC; |
459 | int decaps = 0; |
460 | int async = 0; |
461 | bool xfrm_gro = false; |
462 | bool crypto_done = false; |
463 | struct xfrm_offload *xo = xfrm_offload(skb); |
464 | struct sec_path *sp; |
465 | |
466 | if (encap_type < 0 || (xo && xo->flags & XFRM_GRO)) { |
467 | x = xfrm_input_state(skb); |
468 | |
469 | if (unlikely(x->km.state != XFRM_STATE_VALID)) { |
470 | if (x->km.state == XFRM_STATE_ACQ) |
471 | XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); |
472 | else |
473 | XFRM_INC_STATS(net, |
474 | LINUX_MIB_XFRMINSTATEINVALID); |
475 | |
476 | if (encap_type == -1) |
477 | dev_put(dev: skb->dev); |
478 | goto drop; |
479 | } |
480 | |
481 | family = x->props.family; |
482 | |
483 | /* An encap_type of -1 indicates async resumption. */ |
484 | if (encap_type == -1) { |
485 | async = 1; |
486 | seq = XFRM_SKB_CB(skb)->seq.input.low; |
487 | goto resume; |
488 | } |
489 | /* GRO call */ |
490 | seq = XFRM_SPI_SKB_CB(skb)->seq; |
491 | |
492 | if (xo && (xo->flags & CRYPTO_DONE)) { |
493 | crypto_done = true; |
494 | family = XFRM_SPI_SKB_CB(skb)->family; |
495 | |
496 | if (!(xo->status & CRYPTO_SUCCESS)) { |
497 | if (xo->status & |
498 | (CRYPTO_TRANSPORT_AH_AUTH_FAILED | |
499 | CRYPTO_TRANSPORT_ESP_AUTH_FAILED | |
500 | CRYPTO_TUNNEL_AH_AUTH_FAILED | |
501 | CRYPTO_TUNNEL_ESP_AUTH_FAILED)) { |
502 | |
503 | xfrm_audit_state_icvfail(x, skb, |
504 | proto: x->type->proto); |
505 | x->stats.integrity_failed++; |
506 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); |
507 | goto drop; |
508 | } |
509 | |
510 | if (xo->status & CRYPTO_INVALID_PROTOCOL) { |
511 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); |
512 | goto drop; |
513 | } |
514 | |
515 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
516 | goto drop; |
517 | } |
518 | |
519 | if (xfrm_parse_spi(skb, nexthdr, &spi, &seq)) { |
520 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
521 | goto drop; |
522 | } |
523 | } |
524 | |
525 | goto lock; |
526 | } |
527 | |
528 | family = XFRM_SPI_SKB_CB(skb)->family; |
529 | |
530 | /* if tunnel is present override skb->mark value with tunnel i_key */ |
531 | switch (family) { |
532 | case AF_INET: |
533 | if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4) |
534 | mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key); |
535 | break; |
536 | case AF_INET6: |
537 | if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6) |
538 | mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key); |
539 | break; |
540 | } |
541 | |
542 | sp = secpath_set(skb); |
543 | if (!sp) { |
544 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR); |
545 | goto drop; |
546 | } |
547 | |
548 | seq = 0; |
549 | if (!spi && xfrm_parse_spi(skb, nexthdr, &spi, &seq)) { |
550 | secpath_reset(skb); |
551 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
552 | goto drop; |
553 | } |
554 | |
555 | daddr = (xfrm_address_t *)(skb_network_header(skb) + |
556 | XFRM_SPI_SKB_CB(skb)->daddroff); |
557 | do { |
558 | sp = skb_sec_path(skb); |
559 | |
560 | if (sp->len == XFRM_MAX_DEPTH) { |
561 | secpath_reset(skb); |
562 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
563 | goto drop; |
564 | } |
565 | |
566 | x = xfrm_state_lookup(net, mark, daddr, spi, proto: nexthdr, family); |
567 | if (x == NULL) { |
568 | secpath_reset(skb); |
569 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES); |
570 | xfrm_audit_state_notfound(skb, family, net_spi: spi, net_seq: seq); |
571 | goto drop; |
572 | } |
573 | |
574 | skb->mark = xfrm_smark_get(mark: skb->mark, x); |
575 | |
576 | sp->xvec[sp->len++] = x; |
577 | |
578 | skb_dst_force(skb); |
579 | if (!skb_dst(skb)) { |
580 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR); |
581 | goto drop; |
582 | } |
583 | |
584 | lock: |
585 | spin_lock(lock: &x->lock); |
586 | |
587 | if (unlikely(x->km.state != XFRM_STATE_VALID)) { |
588 | if (x->km.state == XFRM_STATE_ACQ) |
589 | XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); |
590 | else |
591 | XFRM_INC_STATS(net, |
592 | LINUX_MIB_XFRMINSTATEINVALID); |
593 | goto drop_unlock; |
594 | } |
595 | |
596 | if ((x->encap ? x->encap->encap_type : 0) != encap_type) { |
597 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH); |
598 | goto drop_unlock; |
599 | } |
600 | |
601 | if (xfrm_replay_check(x, skb, net_seq: seq)) { |
602 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); |
603 | goto drop_unlock; |
604 | } |
605 | |
606 | if (xfrm_state_check_expire(x)) { |
607 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED); |
608 | goto drop_unlock; |
609 | } |
610 | |
611 | spin_unlock(lock: &x->lock); |
612 | |
613 | if (xfrm_tunnel_check(skb, x, family)) { |
614 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); |
615 | goto drop; |
616 | } |
617 | |
618 | seq_hi = htonl(xfrm_replay_seqhi(x, seq)); |
619 | |
620 | XFRM_SKB_CB(skb)->seq.input.low = seq; |
621 | XFRM_SKB_CB(skb)->seq.input.hi = seq_hi; |
622 | |
623 | dev_hold(dev: skb->dev); |
624 | |
625 | if (crypto_done) |
626 | nexthdr = x->type_offload->input_tail(x, skb); |
627 | else |
628 | nexthdr = x->type->input(x, skb); |
629 | |
630 | if (nexthdr == -EINPROGRESS) |
631 | return 0; |
632 | resume: |
633 | dev_put(dev: skb->dev); |
634 | |
635 | spin_lock(lock: &x->lock); |
636 | if (nexthdr < 0) { |
637 | if (nexthdr == -EBADMSG) { |
638 | xfrm_audit_state_icvfail(x, skb, |
639 | proto: x->type->proto); |
640 | x->stats.integrity_failed++; |
641 | } |
642 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); |
643 | goto drop_unlock; |
644 | } |
645 | |
646 | /* only the first xfrm gets the encap type */ |
647 | encap_type = 0; |
648 | |
649 | if (xfrm_replay_recheck(x, skb, net_seq: seq)) { |
650 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); |
651 | goto drop_unlock; |
652 | } |
653 | |
654 | xfrm_replay_advance(x, net_seq: seq); |
655 | |
656 | x->curlft.bytes += skb->len; |
657 | x->curlft.packets++; |
658 | x->lastused = ktime_get_real_seconds(); |
659 | |
660 | spin_unlock(lock: &x->lock); |
661 | |
662 | XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; |
663 | |
664 | if (xfrm_inner_mode_input(x, skb)) { |
665 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); |
666 | goto drop; |
667 | } |
668 | |
669 | if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) { |
670 | decaps = 1; |
671 | break; |
672 | } |
673 | |
674 | /* |
675 | * We need the inner address. However, we only get here for |
676 | * transport mode so the outer address is identical. |
677 | */ |
678 | daddr = &x->id.daddr; |
679 | family = x->props.family; |
680 | |
681 | err = xfrm_parse_spi(skb, nexthdr, &spi, &seq); |
682 | if (err < 0) { |
683 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
684 | goto drop; |
685 | } |
686 | crypto_done = false; |
687 | } while (!err); |
688 | |
689 | err = xfrm_rcv_cb(skb, family, protocol: x->type->proto, err: 0); |
690 | if (err) |
691 | goto drop; |
692 | |
693 | nf_reset_ct(skb); |
694 | |
695 | if (decaps) { |
696 | sp = skb_sec_path(skb); |
697 | if (sp) |
698 | sp->olen = 0; |
699 | if (skb_valid_dst(skb)) |
700 | skb_dst_drop(skb); |
701 | gro_cells_receive(gcells: &gro_cells, skb); |
702 | return 0; |
703 | } else { |
704 | xo = xfrm_offload(skb); |
705 | if (xo) |
706 | xfrm_gro = xo->flags & XFRM_GRO; |
707 | |
708 | err = -EAFNOSUPPORT; |
709 | rcu_read_lock(); |
710 | afinfo = xfrm_state_afinfo_get_rcu(family: x->props.family); |
711 | if (likely(afinfo)) |
712 | err = afinfo->transport_finish(skb, xfrm_gro || async); |
713 | rcu_read_unlock(); |
714 | if (xfrm_gro) { |
715 | sp = skb_sec_path(skb); |
716 | if (sp) |
717 | sp->olen = 0; |
718 | if (skb_valid_dst(skb)) |
719 | skb_dst_drop(skb); |
720 | gro_cells_receive(gcells: &gro_cells, skb); |
721 | return err; |
722 | } |
723 | |
724 | return err; |
725 | } |
726 | |
727 | drop_unlock: |
728 | spin_unlock(lock: &x->lock); |
729 | drop: |
730 | xfrm_rcv_cb(skb, family, protocol: x && x->type ? x->type->proto : nexthdr, err: -1); |
731 | kfree_skb(skb); |
732 | return 0; |
733 | } |
734 | EXPORT_SYMBOL(xfrm_input); |
735 | |
736 | int xfrm_input_resume(struct sk_buff *skb, int nexthdr) |
737 | { |
738 | return xfrm_input(skb, nexthdr, 0, -1); |
739 | } |
740 | EXPORT_SYMBOL(xfrm_input_resume); |
741 | |
742 | static void xfrm_trans_reinject(struct work_struct *work) |
743 | { |
744 | struct xfrm_trans_tasklet *trans = container_of(work, struct xfrm_trans_tasklet, work); |
745 | struct sk_buff_head queue; |
746 | struct sk_buff *skb; |
747 | |
748 | __skb_queue_head_init(list: &queue); |
749 | spin_lock_bh(lock: &trans->queue_lock); |
750 | skb_queue_splice_init(list: &trans->queue, head: &queue); |
751 | spin_unlock_bh(lock: &trans->queue_lock); |
752 | |
753 | local_bh_disable(); |
754 | while ((skb = __skb_dequeue(list: &queue))) |
755 | XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net, |
756 | NULL, skb); |
757 | local_bh_enable(); |
758 | } |
759 | |
760 | int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb, |
761 | int (*finish)(struct net *, struct sock *, |
762 | struct sk_buff *)) |
763 | { |
764 | struct xfrm_trans_tasklet *trans; |
765 | |
766 | trans = this_cpu_ptr(&xfrm_trans_tasklet); |
767 | |
768 | if (skb_queue_len(list_: &trans->queue) >= READ_ONCE(net_hotdata.max_backlog)) |
769 | return -ENOBUFS; |
770 | |
771 | BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb)); |
772 | |
773 | XFRM_TRANS_SKB_CB(skb)->finish = finish; |
774 | XFRM_TRANS_SKB_CB(skb)->net = net; |
775 | spin_lock_bh(lock: &trans->queue_lock); |
776 | __skb_queue_tail(list: &trans->queue, newsk: skb); |
777 | spin_unlock_bh(lock: &trans->queue_lock); |
778 | schedule_work(work: &trans->work); |
779 | return 0; |
780 | } |
781 | EXPORT_SYMBOL(xfrm_trans_queue_net); |
782 | |
783 | int xfrm_trans_queue(struct sk_buff *skb, |
784 | int (*finish)(struct net *, struct sock *, |
785 | struct sk_buff *)) |
786 | { |
787 | return xfrm_trans_queue_net(dev_net(dev: skb->dev), skb, finish); |
788 | } |
789 | EXPORT_SYMBOL(xfrm_trans_queue); |
790 | |
791 | void __init xfrm_input_init(void) |
792 | { |
793 | int err; |
794 | int i; |
795 | |
796 | init_dummy_netdev(dev: &xfrm_napi_dev); |
797 | err = gro_cells_init(gcells: &gro_cells, dev: &xfrm_napi_dev); |
798 | if (err) |
799 | gro_cells.cells = NULL; |
800 | |
801 | for_each_possible_cpu(i) { |
802 | struct xfrm_trans_tasklet *trans; |
803 | |
804 | trans = &per_cpu(xfrm_trans_tasklet, i); |
805 | spin_lock_init(&trans->queue_lock); |
806 | __skb_queue_head_init(list: &trans->queue); |
807 | INIT_WORK(&trans->work, xfrm_trans_reinject); |
808 | } |
809 | } |
810 | |