1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. |
4 | */ |
5 | |
6 | #include "device.h" |
7 | #include "peer.h" |
8 | #include "socket.h" |
9 | #include "queueing.h" |
10 | #include "messages.h" |
11 | |
12 | #include <linux/ctype.h> |
13 | #include <linux/net.h> |
14 | #include <linux/if_vlan.h> |
15 | #include <linux/if_ether.h> |
16 | #include <linux/inetdevice.h> |
17 | #include <net/udp_tunnel.h> |
18 | #include <net/ipv6.h> |
19 | |
20 | static int send4(struct wg_device *wg, struct sk_buff *skb, |
21 | struct endpoint *endpoint, u8 ds, struct dst_cache *cache) |
22 | { |
23 | struct flowi4 fl = { |
24 | .saddr = endpoint->src4.s_addr, |
25 | .daddr = endpoint->addr4.sin_addr.s_addr, |
26 | .fl4_dport = endpoint->addr4.sin_port, |
27 | .flowi4_mark = wg->fwmark, |
28 | .flowi4_proto = IPPROTO_UDP |
29 | }; |
30 | struct rtable *rt = NULL; |
31 | struct sock *sock; |
32 | int ret = 0; |
33 | |
34 | skb_mark_not_on_list(skb); |
35 | skb->dev = wg->dev; |
36 | skb->mark = wg->fwmark; |
37 | |
38 | rcu_read_lock_bh(); |
39 | sock = rcu_dereference_bh(wg->sock4); |
40 | |
41 | if (unlikely(!sock)) { |
42 | ret = -ENONET; |
43 | goto err; |
44 | } |
45 | |
46 | fl.fl4_sport = inet_sk(sock)->inet_sport; |
47 | |
48 | if (cache) |
49 | rt = dst_cache_get_ip4(dst_cache: cache, saddr: &fl.saddr); |
50 | |
51 | if (!rt) { |
52 | security_sk_classify_flow(sk: sock, flic: flowi4_to_flowi_common(fl4: &fl)); |
53 | if (unlikely(!inet_confirm_addr(sock_net(sock), NULL, 0, |
54 | fl.saddr, RT_SCOPE_HOST))) { |
55 | endpoint->src4.s_addr = 0; |
56 | endpoint->src_if4 = 0; |
57 | fl.saddr = 0; |
58 | if (cache) |
59 | dst_cache_reset(dst_cache: cache); |
60 | } |
61 | rt = ip_route_output_flow(sock_net(sk: sock), flp: &fl, sk: sock); |
62 | if (unlikely(endpoint->src_if4 && ((IS_ERR(rt) && |
63 | PTR_ERR(rt) == -EINVAL) || (!IS_ERR(rt) && |
64 | rt->dst.dev->ifindex != endpoint->src_if4)))) { |
65 | endpoint->src4.s_addr = 0; |
66 | endpoint->src_if4 = 0; |
67 | fl.saddr = 0; |
68 | if (cache) |
69 | dst_cache_reset(dst_cache: cache); |
70 | if (!IS_ERR(ptr: rt)) |
71 | ip_rt_put(rt); |
72 | rt = ip_route_output_flow(sock_net(sk: sock), flp: &fl, sk: sock); |
73 | } |
74 | if (IS_ERR(ptr: rt)) { |
75 | ret = PTR_ERR(ptr: rt); |
76 | net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n" , |
77 | wg->dev->name, &endpoint->addr, ret); |
78 | goto err; |
79 | } |
80 | if (cache) |
81 | dst_cache_set_ip4(dst_cache: cache, dst: &rt->dst, saddr: fl.saddr); |
82 | } |
83 | |
84 | skb->ignore_df = 1; |
85 | udp_tunnel_xmit_skb(rt, sk: sock, skb, src: fl.saddr, dst: fl.daddr, tos: ds, |
86 | ttl: ip4_dst_hoplimit(dst: &rt->dst), df: 0, src_port: fl.fl4_sport, |
87 | dst_port: fl.fl4_dport, xnet: false, nocheck: false); |
88 | goto out; |
89 | |
90 | err: |
91 | kfree_skb(skb); |
92 | out: |
93 | rcu_read_unlock_bh(); |
94 | return ret; |
95 | } |
96 | |
97 | static int send6(struct wg_device *wg, struct sk_buff *skb, |
98 | struct endpoint *endpoint, u8 ds, struct dst_cache *cache) |
99 | { |
100 | #if IS_ENABLED(CONFIG_IPV6) |
101 | struct flowi6 fl = { |
102 | .saddr = endpoint->src6, |
103 | .daddr = endpoint->addr6.sin6_addr, |
104 | .fl6_dport = endpoint->addr6.sin6_port, |
105 | .flowi6_mark = wg->fwmark, |
106 | .flowi6_oif = endpoint->addr6.sin6_scope_id, |
107 | .flowi6_proto = IPPROTO_UDP |
108 | /* TODO: addr->sin6_flowinfo */ |
109 | }; |
110 | struct dst_entry *dst = NULL; |
111 | struct sock *sock; |
112 | int ret = 0; |
113 | |
114 | skb_mark_not_on_list(skb); |
115 | skb->dev = wg->dev; |
116 | skb->mark = wg->fwmark; |
117 | |
118 | rcu_read_lock_bh(); |
119 | sock = rcu_dereference_bh(wg->sock6); |
120 | |
121 | if (unlikely(!sock)) { |
122 | ret = -ENONET; |
123 | goto err; |
124 | } |
125 | |
126 | fl.fl6_sport = inet_sk(sock)->inet_sport; |
127 | |
128 | if (cache) |
129 | dst = dst_cache_get_ip6(dst_cache: cache, saddr: &fl.saddr); |
130 | |
131 | if (!dst) { |
132 | security_sk_classify_flow(sk: sock, flic: flowi6_to_flowi_common(fl6: &fl)); |
133 | if (unlikely(!ipv6_addr_any(&fl.saddr) && |
134 | !ipv6_chk_addr(sock_net(sock), &fl.saddr, NULL, 0))) { |
135 | endpoint->src6 = fl.saddr = in6addr_any; |
136 | if (cache) |
137 | dst_cache_reset(dst_cache: cache); |
138 | } |
139 | dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sk: sock), sock, &fl, |
140 | NULL); |
141 | if (IS_ERR(ptr: dst)) { |
142 | ret = PTR_ERR(ptr: dst); |
143 | net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n" , |
144 | wg->dev->name, &endpoint->addr, ret); |
145 | goto err; |
146 | } |
147 | if (cache) |
148 | dst_cache_set_ip6(dst_cache: cache, dst, saddr: &fl.saddr); |
149 | } |
150 | |
151 | skb->ignore_df = 1; |
152 | udp_tunnel6_xmit_skb(dst, sk: sock, skb, dev: skb->dev, saddr: &fl.saddr, daddr: &fl.daddr, prio: ds, |
153 | ttl: ip6_dst_hoplimit(dst), label: 0, src_port: fl.fl6_sport, |
154 | dst_port: fl.fl6_dport, nocheck: false); |
155 | goto out; |
156 | |
157 | err: |
158 | kfree_skb(skb); |
159 | out: |
160 | rcu_read_unlock_bh(); |
161 | return ret; |
162 | #else |
163 | kfree_skb(skb); |
164 | return -EAFNOSUPPORT; |
165 | #endif |
166 | } |
167 | |
168 | int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, u8 ds) |
169 | { |
170 | size_t skb_len = skb->len; |
171 | int ret = -EAFNOSUPPORT; |
172 | |
173 | read_lock_bh(&peer->endpoint_lock); |
174 | if (peer->endpoint.addr.sa_family == AF_INET) |
175 | ret = send4(wg: peer->device, skb, endpoint: &peer->endpoint, ds, |
176 | cache: &peer->endpoint_cache); |
177 | else if (peer->endpoint.addr.sa_family == AF_INET6) |
178 | ret = send6(wg: peer->device, skb, endpoint: &peer->endpoint, ds, |
179 | cache: &peer->endpoint_cache); |
180 | else |
181 | dev_kfree_skb(skb); |
182 | if (likely(!ret)) |
183 | peer->tx_bytes += skb_len; |
184 | read_unlock_bh(&peer->endpoint_lock); |
185 | |
186 | return ret; |
187 | } |
188 | |
189 | int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *buffer, |
190 | size_t len, u8 ds) |
191 | { |
192 | struct sk_buff *skb = alloc_skb(size: len + SKB_HEADER_LEN, GFP_ATOMIC); |
193 | |
194 | if (unlikely(!skb)) |
195 | return -ENOMEM; |
196 | |
197 | skb_reserve(skb, SKB_HEADER_LEN); |
198 | skb_set_inner_network_header(skb, offset: 0); |
199 | skb_put_data(skb, data: buffer, len); |
200 | return wg_socket_send_skb_to_peer(peer, skb, ds); |
201 | } |
202 | |
203 | int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, |
204 | struct sk_buff *in_skb, void *buffer, |
205 | size_t len) |
206 | { |
207 | int ret = 0; |
208 | struct sk_buff *skb; |
209 | struct endpoint endpoint; |
210 | |
211 | if (unlikely(!in_skb)) |
212 | return -EINVAL; |
213 | ret = wg_socket_endpoint_from_skb(endpoint: &endpoint, skb: in_skb); |
214 | if (unlikely(ret < 0)) |
215 | return ret; |
216 | |
217 | skb = alloc_skb(size: len + SKB_HEADER_LEN, GFP_ATOMIC); |
218 | if (unlikely(!skb)) |
219 | return -ENOMEM; |
220 | skb_reserve(skb, SKB_HEADER_LEN); |
221 | skb_set_inner_network_header(skb, offset: 0); |
222 | skb_put_data(skb, data: buffer, len); |
223 | |
224 | if (endpoint.addr.sa_family == AF_INET) |
225 | ret = send4(wg, skb, endpoint: &endpoint, ds: 0, NULL); |
226 | else if (endpoint.addr.sa_family == AF_INET6) |
227 | ret = send6(wg, skb, endpoint: &endpoint, ds: 0, NULL); |
228 | /* No other possibilities if the endpoint is valid, which it is, |
229 | * as we checked above. |
230 | */ |
231 | |
232 | return ret; |
233 | } |
234 | |
235 | int wg_socket_endpoint_from_skb(struct endpoint *endpoint, |
236 | const struct sk_buff *skb) |
237 | { |
238 | memset(endpoint, 0, sizeof(*endpoint)); |
239 | if (skb->protocol == htons(ETH_P_IP)) { |
240 | endpoint->addr4.sin_family = AF_INET; |
241 | endpoint->addr4.sin_port = udp_hdr(skb)->source; |
242 | endpoint->addr4.sin_addr.s_addr = ip_hdr(skb)->saddr; |
243 | endpoint->src4.s_addr = ip_hdr(skb)->daddr; |
244 | endpoint->src_if4 = skb->skb_iif; |
245 | } else if (IS_ENABLED(CONFIG_IPV6) && skb->protocol == htons(ETH_P_IPV6)) { |
246 | endpoint->addr6.sin6_family = AF_INET6; |
247 | endpoint->addr6.sin6_port = udp_hdr(skb)->source; |
248 | endpoint->addr6.sin6_addr = ipv6_hdr(skb)->saddr; |
249 | endpoint->addr6.sin6_scope_id = ipv6_iface_scope_id( |
250 | addr: &ipv6_hdr(skb)->saddr, iface: skb->skb_iif); |
251 | endpoint->src6 = ipv6_hdr(skb)->daddr; |
252 | } else { |
253 | return -EINVAL; |
254 | } |
255 | return 0; |
256 | } |
257 | |
258 | static bool endpoint_eq(const struct endpoint *a, const struct endpoint *b) |
259 | { |
260 | return (a->addr.sa_family == AF_INET && b->addr.sa_family == AF_INET && |
261 | a->addr4.sin_port == b->addr4.sin_port && |
262 | a->addr4.sin_addr.s_addr == b->addr4.sin_addr.s_addr && |
263 | a->src4.s_addr == b->src4.s_addr && a->src_if4 == b->src_if4) || |
264 | (a->addr.sa_family == AF_INET6 && |
265 | b->addr.sa_family == AF_INET6 && |
266 | a->addr6.sin6_port == b->addr6.sin6_port && |
267 | ipv6_addr_equal(a1: &a->addr6.sin6_addr, a2: &b->addr6.sin6_addr) && |
268 | a->addr6.sin6_scope_id == b->addr6.sin6_scope_id && |
269 | ipv6_addr_equal(a1: &a->src6, a2: &b->src6)) || |
270 | unlikely(!a->addr.sa_family && !b->addr.sa_family); |
271 | } |
272 | |
273 | void wg_socket_set_peer_endpoint(struct wg_peer *peer, |
274 | const struct endpoint *endpoint) |
275 | { |
276 | /* First we check unlocked, in order to optimize, since it's pretty rare |
277 | * that an endpoint will change. If we happen to be mid-write, and two |
278 | * CPUs wind up writing the same thing or something slightly different, |
279 | * it doesn't really matter much either. |
280 | */ |
281 | if (endpoint_eq(a: endpoint, b: &peer->endpoint)) |
282 | return; |
283 | write_lock_bh(&peer->endpoint_lock); |
284 | if (endpoint->addr.sa_family == AF_INET) { |
285 | peer->endpoint.addr4 = endpoint->addr4; |
286 | peer->endpoint.src4 = endpoint->src4; |
287 | peer->endpoint.src_if4 = endpoint->src_if4; |
288 | } else if (IS_ENABLED(CONFIG_IPV6) && endpoint->addr.sa_family == AF_INET6) { |
289 | peer->endpoint.addr6 = endpoint->addr6; |
290 | peer->endpoint.src6 = endpoint->src6; |
291 | } else { |
292 | goto out; |
293 | } |
294 | dst_cache_reset(dst_cache: &peer->endpoint_cache); |
295 | out: |
296 | write_unlock_bh(&peer->endpoint_lock); |
297 | } |
298 | |
299 | void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, |
300 | const struct sk_buff *skb) |
301 | { |
302 | struct endpoint endpoint; |
303 | |
304 | if (!wg_socket_endpoint_from_skb(endpoint: &endpoint, skb)) |
305 | wg_socket_set_peer_endpoint(peer, endpoint: &endpoint); |
306 | } |
307 | |
308 | void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer) |
309 | { |
310 | write_lock_bh(&peer->endpoint_lock); |
311 | memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6)); |
312 | dst_cache_reset_now(dst_cache: &peer->endpoint_cache); |
313 | write_unlock_bh(&peer->endpoint_lock); |
314 | } |
315 | |
316 | static int wg_receive(struct sock *sk, struct sk_buff *skb) |
317 | { |
318 | struct wg_device *wg; |
319 | |
320 | if (unlikely(!sk)) |
321 | goto err; |
322 | wg = sk->sk_user_data; |
323 | if (unlikely(!wg)) |
324 | goto err; |
325 | skb_mark_not_on_list(skb); |
326 | wg_packet_receive(wg, skb); |
327 | return 0; |
328 | |
329 | err: |
330 | kfree_skb(skb); |
331 | return 0; |
332 | } |
333 | |
334 | static void sock_free(struct sock *sock) |
335 | { |
336 | if (unlikely(!sock)) |
337 | return; |
338 | sk_clear_memalloc(sk: sock); |
339 | udp_tunnel_sock_release(sock: sock->sk_socket); |
340 | } |
341 | |
342 | static void set_sock_opts(struct socket *sock) |
343 | { |
344 | sock->sk->sk_allocation = GFP_ATOMIC; |
345 | sock->sk->sk_sndbuf = INT_MAX; |
346 | sk_set_memalloc(sk: sock->sk); |
347 | } |
348 | |
349 | int wg_socket_init(struct wg_device *wg, u16 port) |
350 | { |
351 | struct net *net; |
352 | int ret; |
353 | struct udp_tunnel_sock_cfg cfg = { |
354 | .sk_user_data = wg, |
355 | .encap_type = 1, |
356 | .encap_rcv = wg_receive |
357 | }; |
358 | struct socket *new4 = NULL, *new6 = NULL; |
359 | struct udp_port_cfg port4 = { |
360 | .family = AF_INET, |
361 | .local_ip.s_addr = htonl(INADDR_ANY), |
362 | .local_udp_port = htons(port), |
363 | .use_udp_checksums = true |
364 | }; |
365 | #if IS_ENABLED(CONFIG_IPV6) |
366 | int retries = 0; |
367 | struct udp_port_cfg port6 = { |
368 | .family = AF_INET6, |
369 | .local_ip6 = IN6ADDR_ANY_INIT, |
370 | .use_udp6_tx_checksums = true, |
371 | .use_udp6_rx_checksums = true, |
372 | .ipv6_v6only = true |
373 | }; |
374 | #endif |
375 | |
376 | rcu_read_lock(); |
377 | net = rcu_dereference(wg->creating_net); |
378 | net = net ? maybe_get_net(net) : NULL; |
379 | rcu_read_unlock(); |
380 | if (unlikely(!net)) |
381 | return -ENONET; |
382 | |
383 | #if IS_ENABLED(CONFIG_IPV6) |
384 | retry: |
385 | #endif |
386 | |
387 | ret = udp_sock_create(net, cfg: &port4, sockp: &new4); |
388 | if (ret < 0) { |
389 | pr_err("%s: Could not create IPv4 socket\n" , wg->dev->name); |
390 | goto out; |
391 | } |
392 | set_sock_opts(new4); |
393 | setup_udp_tunnel_sock(net, sock: new4, sock_cfg: &cfg); |
394 | |
395 | #if IS_ENABLED(CONFIG_IPV6) |
396 | if (ipv6_mod_enabled()) { |
397 | port6.local_udp_port = inet_sk(new4->sk)->inet_sport; |
398 | ret = udp_sock_create(net, cfg: &port6, sockp: &new6); |
399 | if (ret < 0) { |
400 | udp_tunnel_sock_release(sock: new4); |
401 | if (ret == -EADDRINUSE && !port && retries++ < 100) |
402 | goto retry; |
403 | pr_err("%s: Could not create IPv6 socket\n" , |
404 | wg->dev->name); |
405 | goto out; |
406 | } |
407 | set_sock_opts(new6); |
408 | setup_udp_tunnel_sock(net, sock: new6, sock_cfg: &cfg); |
409 | } |
410 | #endif |
411 | |
412 | wg_socket_reinit(wg, new4: new4->sk, new6: new6 ? new6->sk : NULL); |
413 | ret = 0; |
414 | out: |
415 | put_net(net); |
416 | return ret; |
417 | } |
418 | |
419 | void wg_socket_reinit(struct wg_device *wg, struct sock *new4, |
420 | struct sock *new6) |
421 | { |
422 | struct sock *old4, *old6; |
423 | |
424 | mutex_lock(&wg->socket_update_lock); |
425 | old4 = rcu_dereference_protected(wg->sock4, |
426 | lockdep_is_held(&wg->socket_update_lock)); |
427 | old6 = rcu_dereference_protected(wg->sock6, |
428 | lockdep_is_held(&wg->socket_update_lock)); |
429 | rcu_assign_pointer(wg->sock4, new4); |
430 | rcu_assign_pointer(wg->sock6, new6); |
431 | if (new4) |
432 | wg->incoming_port = ntohs(inet_sk(new4)->inet_sport); |
433 | mutex_unlock(lock: &wg->socket_update_lock); |
434 | synchronize_net(); |
435 | sock_free(sock: old4); |
436 | sock_free(sock: old6); |
437 | } |
438 | |