1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. |
4 | */ |
5 | |
6 | #include "queueing.h" |
7 | #include "device.h" |
8 | #include "peer.h" |
9 | #include "timers.h" |
10 | #include "messages.h" |
11 | #include "cookie.h" |
12 | #include "socket.h" |
13 | |
14 | #include <linux/ip.h> |
15 | #include <linux/ipv6.h> |
16 | #include <linux/udp.h> |
17 | #include <net/ip_tunnels.h> |
18 | |
19 | /* Must be called with bh disabled. */ |
20 | static void update_rx_stats(struct wg_peer *peer, size_t len) |
21 | { |
22 | dev_sw_netstats_rx_add(dev: peer->device->dev, len); |
23 | peer->rx_bytes += len; |
24 | } |
25 | |
26 | #define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) |
27 | |
28 | static size_t (struct sk_buff *skb) |
29 | { |
30 | if (unlikely(skb->len < sizeof(struct message_header))) |
31 | return 0; |
32 | if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && |
33 | skb->len >= MESSAGE_MINIMUM_LENGTH) |
34 | return sizeof(struct message_data); |
35 | if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && |
36 | skb->len == sizeof(struct message_handshake_initiation)) |
37 | return sizeof(struct message_handshake_initiation); |
38 | if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && |
39 | skb->len == sizeof(struct message_handshake_response)) |
40 | return sizeof(struct message_handshake_response); |
41 | if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && |
42 | skb->len == sizeof(struct message_handshake_cookie)) |
43 | return sizeof(struct message_handshake_cookie); |
44 | return 0; |
45 | } |
46 | |
47 | static int (struct sk_buff *skb, struct wg_device *wg) |
48 | { |
49 | size_t data_offset, data_len, ; |
50 | struct udphdr *udp; |
51 | |
52 | if (unlikely(!wg_check_packet_protocol(skb) || |
53 | skb_transport_header(skb) < skb->head || |
54 | (skb_transport_header(skb) + sizeof(struct udphdr)) > |
55 | skb_tail_pointer(skb))) |
56 | return -EINVAL; /* Bogus IP header */ |
57 | udp = udp_hdr(skb); |
58 | data_offset = (u8 *)udp - skb->data; |
59 | if (unlikely(data_offset > U16_MAX || |
60 | data_offset + sizeof(struct udphdr) > skb->len)) |
61 | /* Packet has offset at impossible location or isn't big enough |
62 | * to have UDP fields. |
63 | */ |
64 | return -EINVAL; |
65 | data_len = ntohs(udp->len); |
66 | if (unlikely(data_len < sizeof(struct udphdr) || |
67 | data_len > skb->len - data_offset)) |
68 | /* UDP packet is reporting too small of a size or lying about |
69 | * its size. |
70 | */ |
71 | return -EINVAL; |
72 | data_len -= sizeof(struct udphdr); |
73 | data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data; |
74 | if (unlikely(!pskb_may_pull(skb, |
75 | data_offset + sizeof(struct message_header)) || |
76 | pskb_trim(skb, data_len + data_offset) < 0)) |
77 | return -EINVAL; |
78 | skb_pull(skb, len: data_offset); |
79 | if (unlikely(skb->len != data_len)) |
80 | /* Final len does not agree with calculated len */ |
81 | return -EINVAL; |
82 | header_len = validate_header_len(skb); |
83 | if (unlikely(!header_len)) |
84 | return -EINVAL; |
85 | __skb_push(skb, len: data_offset); |
86 | if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) |
87 | return -EINVAL; |
88 | __skb_pull(skb, len: data_offset); |
89 | return 0; |
90 | } |
91 | |
92 | static void wg_receive_handshake_packet(struct wg_device *wg, |
93 | struct sk_buff *skb) |
94 | { |
95 | enum cookie_mac_state mac_state; |
96 | struct wg_peer *peer = NULL; |
97 | /* This is global, so that our load calculation applies to the whole |
98 | * system. We don't care about races with it at all. |
99 | */ |
100 | static u64 last_under_load; |
101 | bool packet_needs_cookie; |
102 | bool under_load; |
103 | |
104 | if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { |
105 | net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n" , |
106 | wg->dev->name, skb); |
107 | wg_cookie_message_consume( |
108 | src: (struct message_handshake_cookie *)skb->data, wg); |
109 | return; |
110 | } |
111 | |
112 | under_load = atomic_read(v: &wg->handshake_queue_len) >= |
113 | MAX_QUEUED_INCOMING_HANDSHAKES / 8; |
114 | if (under_load) { |
115 | last_under_load = ktime_get_coarse_boottime_ns(); |
116 | } else if (last_under_load) { |
117 | under_load = !wg_birthdate_has_expired(birthday_nanoseconds: last_under_load, expiration_seconds: 1); |
118 | if (!under_load) |
119 | last_under_load = 0; |
120 | } |
121 | mac_state = wg_cookie_validate_packet(checker: &wg->cookie_checker, skb, |
122 | check_cookie: under_load); |
123 | if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || |
124 | (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) { |
125 | packet_needs_cookie = false; |
126 | } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) { |
127 | packet_needs_cookie = true; |
128 | } else { |
129 | net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n" , |
130 | wg->dev->name, skb); |
131 | return; |
132 | } |
133 | |
134 | switch (SKB_TYPE_LE32(skb)) { |
135 | case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { |
136 | struct message_handshake_initiation *message = |
137 | (struct message_handshake_initiation *)skb->data; |
138 | |
139 | if (packet_needs_cookie) { |
140 | wg_packet_send_handshake_cookie(wg, initiating_skb: skb, |
141 | sender_index: message->sender_index); |
142 | return; |
143 | } |
144 | peer = wg_noise_handshake_consume_initiation(src: message, wg); |
145 | if (unlikely(!peer)) { |
146 | net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n" , |
147 | wg->dev->name, skb); |
148 | return; |
149 | } |
150 | wg_socket_set_peer_endpoint_from_skb(peer, skb); |
151 | net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n" , |
152 | wg->dev->name, peer->internal_id, |
153 | &peer->endpoint.addr); |
154 | wg_packet_send_handshake_response(peer); |
155 | break; |
156 | } |
157 | case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { |
158 | struct message_handshake_response *message = |
159 | (struct message_handshake_response *)skb->data; |
160 | |
161 | if (packet_needs_cookie) { |
162 | wg_packet_send_handshake_cookie(wg, initiating_skb: skb, |
163 | sender_index: message->sender_index); |
164 | return; |
165 | } |
166 | peer = wg_noise_handshake_consume_response(src: message, wg); |
167 | if (unlikely(!peer)) { |
168 | net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n" , |
169 | wg->dev->name, skb); |
170 | return; |
171 | } |
172 | wg_socket_set_peer_endpoint_from_skb(peer, skb); |
173 | net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n" , |
174 | wg->dev->name, peer->internal_id, |
175 | &peer->endpoint.addr); |
176 | if (wg_noise_handshake_begin_session(handshake: &peer->handshake, |
177 | keypairs: &peer->keypairs)) { |
178 | wg_timers_session_derived(peer); |
179 | wg_timers_handshake_complete(peer); |
180 | /* Calling this function will either send any existing |
181 | * packets in the queue and not send a keepalive, which |
182 | * is the best case, Or, if there's nothing in the |
183 | * queue, it will send a keepalive, in order to give |
184 | * immediate confirmation of the session. |
185 | */ |
186 | wg_packet_send_keepalive(peer); |
187 | } |
188 | break; |
189 | } |
190 | } |
191 | |
192 | if (unlikely(!peer)) { |
193 | WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n" ); |
194 | return; |
195 | } |
196 | |
197 | local_bh_disable(); |
198 | update_rx_stats(peer, len: skb->len); |
199 | local_bh_enable(); |
200 | |
201 | wg_timers_any_authenticated_packet_received(peer); |
202 | wg_timers_any_authenticated_packet_traversal(peer); |
203 | wg_peer_put(peer); |
204 | } |
205 | |
206 | void wg_packet_handshake_receive_worker(struct work_struct *work) |
207 | { |
208 | struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr; |
209 | struct wg_device *wg = container_of(queue, struct wg_device, handshake_queue); |
210 | struct sk_buff *skb; |
211 | |
212 | while ((skb = ptr_ring_consume_bh(r: &queue->ring)) != NULL) { |
213 | wg_receive_handshake_packet(wg, skb); |
214 | dev_kfree_skb(skb); |
215 | atomic_dec(v: &wg->handshake_queue_len); |
216 | cond_resched(); |
217 | } |
218 | } |
219 | |
220 | static void keep_key_fresh(struct wg_peer *peer) |
221 | { |
222 | struct noise_keypair *keypair; |
223 | bool send; |
224 | |
225 | if (peer->sent_lastminute_handshake) |
226 | return; |
227 | |
228 | rcu_read_lock_bh(); |
229 | keypair = rcu_dereference_bh(peer->keypairs.current_keypair); |
230 | send = keypair && READ_ONCE(keypair->sending.is_valid) && |
231 | keypair->i_am_the_initiator && |
232 | wg_birthdate_has_expired(birthday_nanoseconds: keypair->sending.birthdate, |
233 | expiration_seconds: REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT); |
234 | rcu_read_unlock_bh(); |
235 | |
236 | if (unlikely(send)) { |
237 | peer->sent_lastminute_handshake = true; |
238 | wg_packet_send_queued_handshake_initiation(peer, is_retry: false); |
239 | } |
240 | } |
241 | |
242 | static bool decrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) |
243 | { |
244 | struct scatterlist sg[MAX_SKB_FRAGS + 8]; |
245 | struct sk_buff *trailer; |
246 | unsigned int offset; |
247 | int num_frags; |
248 | |
249 | if (unlikely(!keypair)) |
250 | return false; |
251 | |
252 | if (unlikely(!READ_ONCE(keypair->receiving.is_valid) || |
253 | wg_birthdate_has_expired(keypair->receiving.birthdate, REJECT_AFTER_TIME) || |
254 | READ_ONCE(keypair->receiving_counter.counter) >= REJECT_AFTER_MESSAGES)) { |
255 | WRITE_ONCE(keypair->receiving.is_valid, false); |
256 | return false; |
257 | } |
258 | |
259 | PACKET_CB(skb)->nonce = |
260 | le64_to_cpu(((struct message_data *)skb->data)->counter); |
261 | |
262 | /* We ensure that the network header is part of the packet before we |
263 | * call skb_cow_data, so that there's no chance that data is removed |
264 | * from the skb, so that later we can extract the original endpoint. |
265 | */ |
266 | offset = -skb_network_offset(skb); |
267 | skb_push(skb, len: offset); |
268 | num_frags = skb_cow_data(skb, tailbits: 0, trailer: &trailer); |
269 | offset += sizeof(struct message_data); |
270 | skb_pull(skb, len: offset); |
271 | if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) |
272 | return false; |
273 | |
274 | sg_init_table(sg, num_frags); |
275 | if (skb_to_sgvec(skb, sg, offset: 0, len: skb->len) <= 0) |
276 | return false; |
277 | |
278 | if (!chacha20poly1305_decrypt_sg_inplace(src: sg, src_len: skb->len, NULL, ad_len: 0, |
279 | PACKET_CB(skb)->nonce, |
280 | key: keypair->receiving.key)) |
281 | return false; |
282 | |
283 | /* Another ugly situation of pushing and pulling the header so as to |
284 | * keep endpoint information intact. |
285 | */ |
286 | skb_push(skb, len: offset); |
287 | if (pskb_trim(skb, len: skb->len - noise_encrypted_len(0))) |
288 | return false; |
289 | skb_pull(skb, len: offset); |
290 | |
291 | return true; |
292 | } |
293 | |
294 | /* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ |
295 | static bool counter_validate(struct noise_replay_counter *counter, u64 their_counter) |
296 | { |
297 | unsigned long index, index_current, top, i; |
298 | bool ret = false; |
299 | |
300 | spin_lock_bh(lock: &counter->lock); |
301 | |
302 | if (unlikely(counter->counter >= REJECT_AFTER_MESSAGES + 1 || |
303 | their_counter >= REJECT_AFTER_MESSAGES)) |
304 | goto out; |
305 | |
306 | ++their_counter; |
307 | |
308 | if (unlikely((COUNTER_WINDOW_SIZE + their_counter) < |
309 | counter->counter)) |
310 | goto out; |
311 | |
312 | index = their_counter >> ilog2(BITS_PER_LONG); |
313 | |
314 | if (likely(their_counter > counter->counter)) { |
315 | index_current = counter->counter >> ilog2(BITS_PER_LONG); |
316 | top = min_t(unsigned long, index - index_current, |
317 | COUNTER_BITS_TOTAL / BITS_PER_LONG); |
318 | for (i = 1; i <= top; ++i) |
319 | counter->backtrack[(i + index_current) & |
320 | ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0; |
321 | WRITE_ONCE(counter->counter, their_counter); |
322 | } |
323 | |
324 | index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1; |
325 | ret = !test_and_set_bit(nr: their_counter & (BITS_PER_LONG - 1), |
326 | addr: &counter->backtrack[index]); |
327 | |
328 | out: |
329 | spin_unlock_bh(lock: &counter->lock); |
330 | return ret; |
331 | } |
332 | |
333 | #include "selftest/counter.c" |
334 | |
335 | static void wg_packet_consume_data_done(struct wg_peer *peer, |
336 | struct sk_buff *skb, |
337 | struct endpoint *endpoint) |
338 | { |
339 | struct net_device *dev = peer->device->dev; |
340 | unsigned int len, len_before_trim; |
341 | struct wg_peer *routed_peer; |
342 | |
343 | wg_socket_set_peer_endpoint(peer, endpoint); |
344 | |
345 | if (unlikely(wg_noise_received_with_keypair(&peer->keypairs, |
346 | PACKET_CB(skb)->keypair))) { |
347 | wg_timers_handshake_complete(peer); |
348 | wg_packet_send_staged_packets(peer); |
349 | } |
350 | |
351 | keep_key_fresh(peer); |
352 | |
353 | wg_timers_any_authenticated_packet_received(peer); |
354 | wg_timers_any_authenticated_packet_traversal(peer); |
355 | |
356 | /* A packet with length 0 is a keepalive packet */ |
357 | if (unlikely(!skb->len)) { |
358 | update_rx_stats(peer, message_data_len(0)); |
359 | net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n" , |
360 | dev->name, peer->internal_id, |
361 | &peer->endpoint.addr); |
362 | goto packet_processed; |
363 | } |
364 | |
365 | wg_timers_data_received(peer); |
366 | |
367 | if (unlikely(skb_network_header(skb) < skb->head)) |
368 | goto dishonest_packet_size; |
369 | if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) && |
370 | (ip_hdr(skb)->version == 4 || |
371 | (ip_hdr(skb)->version == 6 && |
372 | pskb_network_may_pull(skb, sizeof(struct ipv6hdr))))))) |
373 | goto dishonest_packet_type; |
374 | |
375 | skb->dev = dev; |
376 | /* We've already verified the Poly1305 auth tag, which means this packet |
377 | * was not modified in transit. We can therefore tell the networking |
378 | * stack that all checksums of every layer of encapsulation have already |
379 | * been checked "by the hardware" and therefore is unnecessary to check |
380 | * again in software. |
381 | */ |
382 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
383 | skb->csum_level = ~0; /* All levels */ |
384 | skb->protocol = ip_tunnel_parse_protocol(skb); |
385 | if (skb->protocol == htons(ETH_P_IP)) { |
386 | len = ntohs(ip_hdr(skb)->tot_len); |
387 | if (unlikely(len < sizeof(struct iphdr))) |
388 | goto dishonest_packet_size; |
389 | INET_ECN_decapsulate(skb, PACKET_CB(skb)->ds, inner: ip_hdr(skb)->tos); |
390 | } else if (skb->protocol == htons(ETH_P_IPV6)) { |
391 | len = ntohs(ipv6_hdr(skb)->payload_len) + |
392 | sizeof(struct ipv6hdr); |
393 | INET_ECN_decapsulate(skb, PACKET_CB(skb)->ds, inner: ipv6_get_dsfield(ipv6h: ipv6_hdr(skb))); |
394 | } else { |
395 | goto dishonest_packet_type; |
396 | } |
397 | |
398 | if (unlikely(len > skb->len)) |
399 | goto dishonest_packet_size; |
400 | len_before_trim = skb->len; |
401 | if (unlikely(pskb_trim(skb, len))) |
402 | goto packet_processed; |
403 | |
404 | routed_peer = wg_allowedips_lookup_src(table: &peer->device->peer_allowedips, |
405 | skb); |
406 | wg_peer_put(peer: routed_peer); /* We don't need the extra reference. */ |
407 | |
408 | if (unlikely(routed_peer != peer)) |
409 | goto dishonest_packet_peer; |
410 | |
411 | napi_gro_receive(napi: &peer->napi, skb); |
412 | update_rx_stats(peer, message_data_len(len_before_trim)); |
413 | return; |
414 | |
415 | dishonest_packet_peer: |
416 | net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n" , |
417 | dev->name, skb, peer->internal_id, |
418 | &peer->endpoint.addr); |
419 | DEV_STATS_INC(dev, rx_errors); |
420 | DEV_STATS_INC(dev, rx_frame_errors); |
421 | goto packet_processed; |
422 | dishonest_packet_type: |
423 | net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n" , |
424 | dev->name, peer->internal_id, &peer->endpoint.addr); |
425 | DEV_STATS_INC(dev, rx_errors); |
426 | DEV_STATS_INC(dev, rx_frame_errors); |
427 | goto packet_processed; |
428 | dishonest_packet_size: |
429 | net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n" , |
430 | dev->name, peer->internal_id, &peer->endpoint.addr); |
431 | DEV_STATS_INC(dev, rx_errors); |
432 | DEV_STATS_INC(dev, rx_length_errors); |
433 | goto packet_processed; |
434 | packet_processed: |
435 | dev_kfree_skb(skb); |
436 | } |
437 | |
438 | int wg_packet_rx_poll(struct napi_struct *napi, int budget) |
439 | { |
440 | struct wg_peer *peer = container_of(napi, struct wg_peer, napi); |
441 | struct noise_keypair *keypair; |
442 | struct endpoint endpoint; |
443 | enum packet_state state; |
444 | struct sk_buff *skb; |
445 | int work_done = 0; |
446 | bool free; |
447 | |
448 | if (unlikely(budget <= 0)) |
449 | return 0; |
450 | |
451 | while ((skb = wg_prev_queue_peek(queue: &peer->rx_queue)) != NULL && |
452 | (state = atomic_read_acquire(v: &PACKET_CB(skb)->state)) != |
453 | PACKET_STATE_UNCRYPTED) { |
454 | wg_prev_queue_drop_peeked(queue: &peer->rx_queue); |
455 | keypair = PACKET_CB(skb)->keypair; |
456 | free = true; |
457 | |
458 | if (unlikely(state != PACKET_STATE_CRYPTED)) |
459 | goto next; |
460 | |
461 | if (unlikely(!counter_validate(&keypair->receiving_counter, |
462 | PACKET_CB(skb)->nonce))) { |
463 | net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n" , |
464 | peer->device->dev->name, |
465 | PACKET_CB(skb)->nonce, |
466 | READ_ONCE(keypair->receiving_counter.counter)); |
467 | goto next; |
468 | } |
469 | |
470 | if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb))) |
471 | goto next; |
472 | |
473 | wg_reset_packet(skb, encapsulating: false); |
474 | wg_packet_consume_data_done(peer, skb, endpoint: &endpoint); |
475 | free = false; |
476 | |
477 | next: |
478 | wg_noise_keypair_put(keypair, unreference_now: false); |
479 | wg_peer_put(peer); |
480 | if (unlikely(free)) |
481 | dev_kfree_skb(skb); |
482 | |
483 | if (++work_done >= budget) |
484 | break; |
485 | } |
486 | |
487 | if (work_done < budget) |
488 | napi_complete_done(n: napi, work_done); |
489 | |
490 | return work_done; |
491 | } |
492 | |
493 | void wg_packet_decrypt_worker(struct work_struct *work) |
494 | { |
495 | struct crypt_queue *queue = container_of(work, struct multicore_worker, |
496 | work)->ptr; |
497 | struct sk_buff *skb; |
498 | |
499 | while ((skb = ptr_ring_consume_bh(r: &queue->ring)) != NULL) { |
500 | enum packet_state state = |
501 | likely(decrypt_packet(skb, PACKET_CB(skb)->keypair)) ? |
502 | PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; |
503 | wg_queue_enqueue_per_peer_rx(skb, state); |
504 | if (need_resched()) |
505 | cond_resched(); |
506 | } |
507 | } |
508 | |
509 | static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) |
510 | { |
511 | __le32 idx = ((struct message_data *)skb->data)->key_idx; |
512 | struct wg_peer *peer = NULL; |
513 | int ret; |
514 | |
515 | rcu_read_lock_bh(); |
516 | PACKET_CB(skb)->keypair = |
517 | (struct noise_keypair *)wg_index_hashtable_lookup( |
518 | table: wg->index_hashtable, type_mask: INDEX_HASHTABLE_KEYPAIR, index: idx, |
519 | peer: &peer); |
520 | if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair))) |
521 | goto err_keypair; |
522 | |
523 | if (unlikely(READ_ONCE(peer->is_dead))) |
524 | goto err; |
525 | |
526 | ret = wg_queue_enqueue_per_device_and_peer(device_queue: &wg->decrypt_queue, peer_queue: &peer->rx_queue, skb, |
527 | wq: wg->packet_crypt_wq); |
528 | if (unlikely(ret == -EPIPE)) |
529 | wg_queue_enqueue_per_peer_rx(skb, state: PACKET_STATE_DEAD); |
530 | if (likely(!ret || ret == -EPIPE)) { |
531 | rcu_read_unlock_bh(); |
532 | return; |
533 | } |
534 | err: |
535 | wg_noise_keypair_put(PACKET_CB(skb)->keypair, unreference_now: false); |
536 | err_keypair: |
537 | rcu_read_unlock_bh(); |
538 | wg_peer_put(peer); |
539 | dev_kfree_skb(skb); |
540 | } |
541 | |
542 | void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) |
543 | { |
544 | if (unlikely(prepare_skb_header(skb, wg) < 0)) |
545 | goto err; |
546 | switch (SKB_TYPE_LE32(skb)) { |
547 | case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): |
548 | case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): |
549 | case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { |
550 | int cpu, ret = -EBUSY; |
551 | |
552 | if (unlikely(!rng_is_initialized())) |
553 | goto drop; |
554 | if (atomic_read(v: &wg->handshake_queue_len) > MAX_QUEUED_INCOMING_HANDSHAKES / 2) { |
555 | if (spin_trylock_bh(lock: &wg->handshake_queue.ring.producer_lock)) { |
556 | ret = __ptr_ring_produce(r: &wg->handshake_queue.ring, ptr: skb); |
557 | spin_unlock_bh(lock: &wg->handshake_queue.ring.producer_lock); |
558 | } |
559 | } else |
560 | ret = ptr_ring_produce_bh(r: &wg->handshake_queue.ring, ptr: skb); |
561 | if (ret) { |
562 | drop: |
563 | net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n" , |
564 | wg->dev->name, skb); |
565 | goto err; |
566 | } |
567 | atomic_inc(v: &wg->handshake_queue_len); |
568 | cpu = wg_cpumask_next_online(last_cpu: &wg->handshake_queue.last_cpu); |
569 | /* Queues up a call to packet_process_queued_handshake_packets(skb): */ |
570 | queue_work_on(cpu, wq: wg->handshake_receive_wq, |
571 | work: &per_cpu_ptr(wg->handshake_queue.worker, cpu)->work); |
572 | break; |
573 | } |
574 | case cpu_to_le32(MESSAGE_DATA): |
575 | PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(iph: ip_hdr(skb), skb); |
576 | wg_packet_consume_data(wg, skb); |
577 | break; |
578 | default: |
579 | WARN(1, "Non-exhaustive parsing of packet header lead to unknown packet type!\n" ); |
580 | goto err; |
581 | } |
582 | return; |
583 | |
584 | err: |
585 | dev_kfree_skb(skb); |
586 | } |
587 | |