1// SPDX-License-Identifier: GPL-2.0
2/* Multipath TCP
3 *
4 * Copyright (c) 2019, Intel Corporation.
5 */
6#define pr_fmt(fmt) "MPTCP: " fmt
7
8#include <linux/rculist.h>
9#include <linux/spinlock.h>
10#include "protocol.h"
11#include "mib.h"
12
13#define ADD_ADDR_RETRANS_MAX 3
14
15struct mptcp_pm_add_entry {
16 struct list_head list;
17 struct mptcp_addr_info addr;
18 u8 retrans_times;
19 struct timer_list add_timer;
20 struct mptcp_sock *sock;
21 struct rcu_head rcu;
22};
23
24static DEFINE_SPINLOCK(mptcp_pm_list_lock);
25static LIST_HEAD(mptcp_pm_list);
26
27/* path manager helpers */
28
29/* if sk is ipv4 or ipv6_only allows only same-family local and remote addresses,
30 * otherwise allow any matching local/remote pair
31 */
32bool mptcp_pm_addr_families_match(const struct sock *sk,
33 const struct mptcp_addr_info *loc,
34 const struct mptcp_addr_info *rem)
35{
36 bool mptcp_is_v4 = sk->sk_family == AF_INET;
37
38#if IS_ENABLED(CONFIG_MPTCP_IPV6)
39 bool loc_is_v4 = loc->family == AF_INET || ipv6_addr_v4mapped(a: &loc->addr6);
40 bool rem_is_v4 = rem->family == AF_INET || ipv6_addr_v4mapped(a: &rem->addr6);
41
42 if (mptcp_is_v4)
43 return loc_is_v4 && rem_is_v4;
44
45 if (ipv6_only_sock(sk))
46 return !loc_is_v4 && !rem_is_v4;
47
48 return loc_is_v4 == rem_is_v4;
49#else
50 return mptcp_is_v4 && loc->family == AF_INET && rem->family == AF_INET;
51#endif
52}
53
54bool mptcp_addresses_equal(const struct mptcp_addr_info *a,
55 const struct mptcp_addr_info *b, bool use_port)
56{
57 bool addr_equals = false;
58
59 if (a->family == b->family) {
60 if (a->family == AF_INET)
61 addr_equals = a->addr.s_addr == b->addr.s_addr;
62#if IS_ENABLED(CONFIG_MPTCP_IPV6)
63 else
64 addr_equals = ipv6_addr_equal(a1: &a->addr6, a2: &b->addr6);
65 } else if (a->family == AF_INET) {
66 if (ipv6_addr_v4mapped(a: &b->addr6))
67 addr_equals = a->addr.s_addr == b->addr6.s6_addr32[3];
68 } else if (b->family == AF_INET) {
69 if (ipv6_addr_v4mapped(a: &a->addr6))
70 addr_equals = a->addr6.s6_addr32[3] == b->addr.s_addr;
71#endif
72 }
73
74 if (!addr_equals)
75 return false;
76 if (!use_port)
77 return true;
78
79 return a->port == b->port;
80}
81
82void mptcp_local_address(const struct sock_common *skc,
83 struct mptcp_addr_info *addr)
84{
85 addr->family = skc->skc_family;
86 addr->port = htons(skc->skc_num);
87 if (addr->family == AF_INET)
88 addr->addr.s_addr = skc->skc_rcv_saddr;
89#if IS_ENABLED(CONFIG_MPTCP_IPV6)
90 else if (addr->family == AF_INET6)
91 addr->addr6 = skc->skc_v6_rcv_saddr;
92#endif
93}
94
95void mptcp_remote_address(const struct sock_common *skc,
96 struct mptcp_addr_info *addr)
97{
98 addr->family = skc->skc_family;
99 addr->port = skc->skc_dport;
100 if (addr->family == AF_INET)
101 addr->addr.s_addr = skc->skc_daddr;
102#if IS_ENABLED(CONFIG_MPTCP_IPV6)
103 else if (addr->family == AF_INET6)
104 addr->addr6 = skc->skc_v6_daddr;
105#endif
106}
107
108static bool mptcp_pm_is_init_remote_addr(struct mptcp_sock *msk,
109 const struct mptcp_addr_info *remote)
110{
111 struct mptcp_addr_info mpc_remote;
112
113 mptcp_remote_address(skc: (struct sock_common *)msk, addr: &mpc_remote);
114 return mptcp_addresses_equal(a: &mpc_remote, b: remote, use_port: remote->port);
115}
116
117bool mptcp_lookup_subflow_by_saddr(const struct list_head *list,
118 const struct mptcp_addr_info *saddr)
119{
120 struct mptcp_subflow_context *subflow;
121 struct mptcp_addr_info cur;
122 struct sock_common *skc;
123
124 list_for_each_entry(subflow, list, node) {
125 skc = (struct sock_common *)mptcp_subflow_tcp_sock(subflow);
126
127 mptcp_local_address(skc, addr: &cur);
128 if (mptcp_addresses_equal(a: &cur, b: saddr, use_port: saddr->port))
129 return true;
130 }
131
132 return false;
133}
134
135static struct mptcp_pm_add_entry *
136mptcp_lookup_anno_list_by_saddr(const struct mptcp_sock *msk,
137 const struct mptcp_addr_info *addr)
138{
139 struct mptcp_pm_add_entry *entry;
140
141 lockdep_assert_held(&msk->pm.lock);
142
143 list_for_each_entry(entry, &msk->pm.anno_list, list) {
144 if (mptcp_addresses_equal(a: &entry->addr, b: addr, use_port: true))
145 return entry;
146 }
147
148 return NULL;
149}
150
151bool mptcp_remove_anno_list_by_saddr(struct mptcp_sock *msk,
152 const struct mptcp_addr_info *addr)
153{
154 struct mptcp_pm_add_entry *entry;
155 bool ret;
156
157 entry = mptcp_pm_del_add_timer(msk, addr, check_id: false);
158 ret = entry;
159 kfree_rcu(entry, rcu);
160
161 return ret;
162}
163
164bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk)
165{
166 struct mptcp_pm_add_entry *entry;
167 struct mptcp_addr_info saddr;
168 bool ret = false;
169
170 mptcp_local_address(skc: (struct sock_common *)sk, addr: &saddr);
171
172 spin_lock_bh(lock: &msk->pm.lock);
173 list_for_each_entry(entry, &msk->pm.anno_list, list) {
174 if (mptcp_addresses_equal(a: &entry->addr, b: &saddr, use_port: true)) {
175 ret = true;
176 goto out;
177 }
178 }
179
180out:
181 spin_unlock_bh(lock: &msk->pm.lock);
182 return ret;
183}
184
185static void __mptcp_pm_send_ack(struct mptcp_sock *msk,
186 struct mptcp_subflow_context *subflow,
187 bool prio, bool backup)
188{
189 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
190 bool slow;
191
192 pr_debug("send ack for %s\n",
193 prio ? "mp_prio" :
194 (mptcp_pm_should_add_signal(msk) ? "add_addr" : "rm_addr"));
195
196 slow = lock_sock_fast(sk: ssk);
197 if (prio) {
198 subflow->send_mp_prio = 1;
199 subflow->request_bkup = backup;
200 }
201
202 __mptcp_subflow_send_ack(ssk);
203 unlock_sock_fast(sk: ssk, slow);
204}
205
206void mptcp_pm_send_ack(struct mptcp_sock *msk,
207 struct mptcp_subflow_context *subflow,
208 bool prio, bool backup)
209{
210 spin_unlock_bh(lock: &msk->pm.lock);
211 __mptcp_pm_send_ack(msk, subflow, prio, backup);
212 spin_lock_bh(lock: &msk->pm.lock);
213}
214
215void mptcp_pm_addr_send_ack(struct mptcp_sock *msk)
216{
217 struct mptcp_subflow_context *subflow, *alt = NULL;
218
219 msk_owned_by_me(msk);
220 lockdep_assert_held(&msk->pm.lock);
221
222 if (!mptcp_pm_should_add_signal(msk) &&
223 !mptcp_pm_should_rm_signal(msk))
224 return;
225
226 mptcp_for_each_subflow(msk, subflow) {
227 if (__mptcp_subflow_active(subflow)) {
228 if (!subflow->stale) {
229 mptcp_pm_send_ack(msk, subflow, prio: false, backup: false);
230 return;
231 }
232
233 if (!alt)
234 alt = subflow;
235 }
236 }
237
238 if (alt)
239 mptcp_pm_send_ack(msk, subflow: alt, prio: false, backup: false);
240}
241
242int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk,
243 struct mptcp_addr_info *addr,
244 struct mptcp_addr_info *rem,
245 u8 bkup)
246{
247 struct mptcp_subflow_context *subflow;
248
249 pr_debug("bkup=%d\n", bkup);
250
251 mptcp_for_each_subflow(msk, subflow) {
252 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
253 struct mptcp_addr_info local, remote;
254
255 mptcp_local_address(skc: (struct sock_common *)ssk, addr: &local);
256 if (!mptcp_addresses_equal(a: &local, b: addr, use_port: addr->port))
257 continue;
258
259 if (rem && rem->family != AF_UNSPEC) {
260 mptcp_remote_address(skc: (struct sock_common *)ssk, addr: &remote);
261 if (!mptcp_addresses_equal(a: &remote, b: rem, use_port: rem->port))
262 continue;
263 }
264
265 __mptcp_pm_send_ack(msk, subflow, prio: true, backup: bkup);
266 return 0;
267 }
268
269 return -EINVAL;
270}
271
272static unsigned int mptcp_adjust_add_addr_timeout(struct mptcp_sock *msk)
273{
274 const struct net *net = sock_net(sk: (struct sock *)msk);
275 unsigned int rto = mptcp_get_add_addr_timeout(net);
276 struct mptcp_subflow_context *subflow;
277 unsigned int max = 0;
278
279 mptcp_for_each_subflow(msk, subflow) {
280 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
281 struct inet_connection_sock *icsk = inet_csk(ssk);
282
283 if (icsk->icsk_rto > max)
284 max = icsk->icsk_rto;
285 }
286
287 if (max && max < rto)
288 rto = max;
289
290 return rto;
291}
292
293static void mptcp_pm_add_timer(struct timer_list *timer)
294{
295 struct mptcp_pm_add_entry *entry = timer_container_of(entry, timer,
296 add_timer);
297 struct mptcp_sock *msk = entry->sock;
298 struct sock *sk = (struct sock *)msk;
299 unsigned int timeout;
300
301 pr_debug("msk=%p\n", msk);
302
303 if (!msk)
304 return;
305
306 if (inet_sk_state_load(sk) == TCP_CLOSE)
307 return;
308
309 if (!entry->addr.id)
310 return;
311
312 if (mptcp_pm_should_add_signal_addr(msk)) {
313 sk_reset_timer(sk, timer, expires: jiffies + TCP_RTO_MAX / 8);
314 goto out;
315 }
316
317 timeout = mptcp_adjust_add_addr_timeout(msk);
318 if (!timeout)
319 goto out;
320
321 spin_lock_bh(lock: &msk->pm.lock);
322
323 if (!mptcp_pm_should_add_signal_addr(msk)) {
324 pr_debug("retransmit ADD_ADDR id=%d\n", entry->addr.id);
325 mptcp_pm_announce_addr(msk, addr: &entry->addr, echo: false);
326 mptcp_pm_add_addr_send_ack(msk);
327 entry->retrans_times++;
328 }
329
330 if (entry->retrans_times < ADD_ADDR_RETRANS_MAX)
331 sk_reset_timer(sk, timer,
332 expires: jiffies + (timeout << entry->retrans_times));
333
334 spin_unlock_bh(lock: &msk->pm.lock);
335
336 if (entry->retrans_times == ADD_ADDR_RETRANS_MAX)
337 mptcp_pm_subflow_established(msk);
338
339out:
340 __sock_put(sk);
341}
342
343struct mptcp_pm_add_entry *
344mptcp_pm_del_add_timer(struct mptcp_sock *msk,
345 const struct mptcp_addr_info *addr, bool check_id)
346{
347 struct mptcp_pm_add_entry *entry;
348 struct sock *sk = (struct sock *)msk;
349 bool stop_timer = false;
350
351 rcu_read_lock();
352
353 spin_lock_bh(lock: &msk->pm.lock);
354 entry = mptcp_lookup_anno_list_by_saddr(msk, addr);
355 if (entry && (!check_id || entry->addr.id == addr->id)) {
356 entry->retrans_times = ADD_ADDR_RETRANS_MAX;
357 stop_timer = true;
358 }
359 if (!check_id && entry)
360 list_del(entry: &entry->list);
361 spin_unlock_bh(lock: &msk->pm.lock);
362
363 /* Note: entry might have been removed by another thread.
364 * We hold rcu_read_lock() to ensure it is not freed under us.
365 */
366 if (stop_timer)
367 sk_stop_timer_sync(sk, timer: &entry->add_timer);
368
369 rcu_read_unlock();
370 return entry;
371}
372
373bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk,
374 const struct mptcp_addr_info *addr)
375{
376 struct mptcp_pm_add_entry *add_entry = NULL;
377 struct sock *sk = (struct sock *)msk;
378 unsigned int timeout;
379
380 lockdep_assert_held(&msk->pm.lock);
381
382 add_entry = mptcp_lookup_anno_list_by_saddr(msk, addr);
383
384 if (add_entry) {
385 if (WARN_ON_ONCE(mptcp_pm_is_kernel(msk)))
386 return false;
387
388 goto reset_timer;
389 }
390
391 add_entry = kmalloc(sizeof(*add_entry), GFP_ATOMIC);
392 if (!add_entry)
393 return false;
394
395 list_add(new: &add_entry->list, head: &msk->pm.anno_list);
396
397 add_entry->addr = *addr;
398 add_entry->sock = msk;
399 add_entry->retrans_times = 0;
400
401 timer_setup(&add_entry->add_timer, mptcp_pm_add_timer, 0);
402reset_timer:
403 timeout = mptcp_adjust_add_addr_timeout(msk);
404 if (timeout)
405 sk_reset_timer(sk, timer: &add_entry->add_timer, expires: jiffies + timeout);
406
407 return true;
408}
409
410static void mptcp_pm_free_anno_list(struct mptcp_sock *msk)
411{
412 struct mptcp_pm_add_entry *entry, *tmp;
413 struct sock *sk = (struct sock *)msk;
414 LIST_HEAD(free_list);
415
416 pr_debug("msk=%p\n", msk);
417
418 spin_lock_bh(lock: &msk->pm.lock);
419 list_splice_init(list: &msk->pm.anno_list, head: &free_list);
420 spin_unlock_bh(lock: &msk->pm.lock);
421
422 list_for_each_entry_safe(entry, tmp, &free_list, list) {
423 sk_stop_timer_sync(sk, timer: &entry->add_timer);
424 kfree_rcu(entry, rcu);
425 }
426}
427
428/* path manager command handlers */
429
430int mptcp_pm_announce_addr(struct mptcp_sock *msk,
431 const struct mptcp_addr_info *addr,
432 bool echo)
433{
434 u8 add_addr = READ_ONCE(msk->pm.addr_signal);
435
436 pr_debug("msk=%p, local_id=%d, echo=%d\n", msk, addr->id, echo);
437
438 lockdep_assert_held(&msk->pm.lock);
439
440 if (add_addr &
441 (echo ? BIT(MPTCP_ADD_ADDR_ECHO) : BIT(MPTCP_ADD_ADDR_SIGNAL))) {
442 MPTCP_INC_STATS(net: sock_net(sk: (struct sock *)msk),
443 field: echo ? MPTCP_MIB_ECHOADDTXDROP : MPTCP_MIB_ADDADDRTXDROP);
444 return -EINVAL;
445 }
446
447 if (echo) {
448 msk->pm.remote = *addr;
449 add_addr |= BIT(MPTCP_ADD_ADDR_ECHO);
450 } else {
451 msk->pm.local = *addr;
452 add_addr |= BIT(MPTCP_ADD_ADDR_SIGNAL);
453 }
454 WRITE_ONCE(msk->pm.addr_signal, add_addr);
455 return 0;
456}
457
458int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list)
459{
460 u8 rm_addr = READ_ONCE(msk->pm.addr_signal);
461
462 pr_debug("msk=%p, rm_list_nr=%d\n", msk, rm_list->nr);
463
464 if (rm_addr) {
465 MPTCP_ADD_STATS(net: sock_net(sk: (struct sock *)msk),
466 field: MPTCP_MIB_RMADDRTXDROP, val: rm_list->nr);
467 return -EINVAL;
468 }
469
470 msk->pm.rm_list_tx = *rm_list;
471 rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL);
472 WRITE_ONCE(msk->pm.addr_signal, rm_addr);
473 mptcp_pm_addr_send_ack(msk);
474 return 0;
475}
476
477/* path manager event handlers */
478
479void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side)
480{
481 struct mptcp_pm_data *pm = &msk->pm;
482
483 pr_debug("msk=%p, token=%u side=%d\n", msk, READ_ONCE(msk->token), server_side);
484
485 WRITE_ONCE(pm->server_side, server_side);
486 mptcp_event(type: MPTCP_EVENT_CREATED, msk, ssk, GFP_ATOMIC);
487}
488
489bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk)
490{
491 struct mptcp_pm_data *pm = &msk->pm;
492 unsigned int limit_extra_subflows;
493 int ret = 0;
494
495 if (mptcp_pm_is_userspace(msk)) {
496 if (mptcp_userspace_pm_active(msk)) {
497 spin_lock_bh(lock: &pm->lock);
498 pm->extra_subflows++;
499 spin_unlock_bh(lock: &pm->lock);
500 return true;
501 }
502 return false;
503 }
504
505 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
506
507 pr_debug("msk=%p subflows=%d max=%d allow=%d\n", msk,
508 pm->extra_subflows, limit_extra_subflows,
509 READ_ONCE(pm->accept_subflow));
510
511 /* try to avoid acquiring the lock below */
512 if (!READ_ONCE(pm->accept_subflow))
513 return false;
514
515 spin_lock_bh(lock: &pm->lock);
516 if (READ_ONCE(pm->accept_subflow)) {
517 ret = pm->extra_subflows < limit_extra_subflows;
518 if (ret && ++pm->extra_subflows == limit_extra_subflows)
519 WRITE_ONCE(pm->accept_subflow, false);
520 }
521 spin_unlock_bh(lock: &pm->lock);
522
523 return ret;
524}
525
526/* return true if the new status bit is currently cleared, that is, this event
527 * can be server, eventually by an already scheduled work
528 */
529static bool mptcp_pm_schedule_work(struct mptcp_sock *msk,
530 enum mptcp_pm_status new_status)
531{
532 pr_debug("msk=%p status=%x new=%lx\n", msk, msk->pm.status,
533 BIT(new_status));
534 if (msk->pm.status & BIT(new_status))
535 return false;
536
537 msk->pm.status |= BIT(new_status);
538 mptcp_schedule_work(sk: (struct sock *)msk);
539 return true;
540}
541
542void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk)
543{
544 struct mptcp_pm_data *pm = &msk->pm;
545 bool announce = false;
546
547 pr_debug("msk=%p\n", msk);
548
549 spin_lock_bh(lock: &pm->lock);
550
551 /* mptcp_pm_fully_established() can be invoked by multiple
552 * racing paths - accept() and check_fully_established()
553 * be sure to serve this event only once.
554 */
555 if (READ_ONCE(pm->work_pending) &&
556 !(pm->status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)))
557 mptcp_pm_schedule_work(msk, new_status: MPTCP_PM_ESTABLISHED);
558
559 if ((pm->status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)) == 0)
560 announce = true;
561
562 pm->status |= BIT(MPTCP_PM_ALREADY_ESTABLISHED);
563 spin_unlock_bh(lock: &pm->lock);
564
565 if (announce)
566 mptcp_event(type: MPTCP_EVENT_ESTABLISHED, msk, ssk, GFP_ATOMIC);
567}
568
569void mptcp_pm_connection_closed(struct mptcp_sock *msk)
570{
571 pr_debug("msk=%p\n", msk);
572
573 if (msk->token)
574 mptcp_event(type: MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL);
575}
576
577void mptcp_pm_subflow_established(struct mptcp_sock *msk)
578{
579 struct mptcp_pm_data *pm = &msk->pm;
580
581 pr_debug("msk=%p\n", msk);
582
583 if (!READ_ONCE(pm->work_pending))
584 return;
585
586 spin_lock_bh(lock: &pm->lock);
587
588 if (READ_ONCE(pm->work_pending))
589 mptcp_pm_schedule_work(msk, new_status: MPTCP_PM_SUBFLOW_ESTABLISHED);
590
591 spin_unlock_bh(lock: &pm->lock);
592}
593
594void mptcp_pm_subflow_check_next(struct mptcp_sock *msk,
595 const struct mptcp_subflow_context *subflow)
596{
597 struct sock *sk = (struct sock *)msk;
598 struct mptcp_pm_data *pm = &msk->pm;
599 bool update_subflows;
600
601 update_subflows = subflow->request_join || subflow->mp_join;
602 if (mptcp_pm_is_userspace(msk)) {
603 if (update_subflows) {
604 spin_lock_bh(lock: &pm->lock);
605 pm->extra_subflows--;
606 spin_unlock_bh(lock: &pm->lock);
607 }
608 return;
609 }
610
611 if (!READ_ONCE(pm->work_pending) && !update_subflows)
612 return;
613
614 spin_lock_bh(lock: &pm->lock);
615 if (update_subflows)
616 __mptcp_pm_close_subflow(msk);
617
618 /* Even if this subflow is not really established, tell the PM to try
619 * to pick the next ones, if possible.
620 */
621 if (mptcp_is_fully_established(sk) &&
622 mptcp_pm_nl_check_work_pending(msk))
623 mptcp_pm_schedule_work(msk, new_status: MPTCP_PM_SUBFLOW_ESTABLISHED);
624
625 spin_unlock_bh(lock: &pm->lock);
626}
627
628void mptcp_pm_add_addr_received(const struct sock *ssk,
629 const struct mptcp_addr_info *addr)
630{
631 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk: ssk);
632 struct mptcp_sock *msk = mptcp_sk(subflow->conn);
633 struct mptcp_pm_data *pm = &msk->pm;
634
635 pr_debug("msk=%p remote_id=%d accept=%d\n", msk, addr->id,
636 READ_ONCE(pm->accept_addr));
637
638 mptcp_event_addr_announced(ssk, info: addr);
639
640 spin_lock_bh(lock: &pm->lock);
641
642 if (mptcp_pm_is_userspace(msk)) {
643 if (mptcp_userspace_pm_active(msk)) {
644 mptcp_pm_announce_addr(msk, addr, echo: true);
645 mptcp_pm_add_addr_send_ack(msk);
646 } else {
647 __MPTCP_INC_STATS(net: sock_net(sk: (struct sock *)msk), field: MPTCP_MIB_ADDADDRDROP);
648 }
649 /* - id0 should not have a different address
650 * - special case for C-flag: linked to fill_local_addresses_vec()
651 */
652 } else if ((addr->id == 0 && !mptcp_pm_is_init_remote_addr(msk, remote: addr)) ||
653 (addr->id > 0 && !READ_ONCE(pm->accept_addr) &&
654 !mptcp_pm_add_addr_c_flag_case(msk))) {
655 mptcp_pm_announce_addr(msk, addr, echo: true);
656 mptcp_pm_add_addr_send_ack(msk);
657 } else if (mptcp_pm_schedule_work(msk, new_status: MPTCP_PM_ADD_ADDR_RECEIVED)) {
658 pm->remote = *addr;
659 } else {
660 __MPTCP_INC_STATS(net: sock_net(sk: (struct sock *)msk), field: MPTCP_MIB_ADDADDRDROP);
661 }
662
663 spin_unlock_bh(lock: &pm->lock);
664}
665
666void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk,
667 const struct mptcp_addr_info *addr)
668{
669 struct mptcp_pm_data *pm = &msk->pm;
670
671 pr_debug("msk=%p\n", msk);
672
673 if (!READ_ONCE(pm->work_pending))
674 return;
675
676 spin_lock_bh(lock: &pm->lock);
677
678 if (mptcp_lookup_anno_list_by_saddr(msk, addr) && READ_ONCE(pm->work_pending))
679 mptcp_pm_schedule_work(msk, new_status: MPTCP_PM_SUBFLOW_ESTABLISHED);
680
681 spin_unlock_bh(lock: &pm->lock);
682}
683
684void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk)
685{
686 if (!mptcp_pm_should_add_signal(msk))
687 return;
688
689 mptcp_pm_schedule_work(msk, new_status: MPTCP_PM_ADD_ADDR_SEND_ACK);
690}
691
692static void mptcp_pm_rm_addr_or_subflow(struct mptcp_sock *msk,
693 const struct mptcp_rm_list *rm_list,
694 enum linux_mptcp_mib_field rm_type)
695{
696 struct mptcp_subflow_context *subflow, *tmp;
697 struct sock *sk = (struct sock *)msk;
698 u8 i;
699
700 pr_debug("%s rm_list_nr %d\n",
701 rm_type == MPTCP_MIB_RMADDR ? "address" : "subflow", rm_list->nr);
702
703 msk_owned_by_me(msk);
704
705 if (sk->sk_state == TCP_LISTEN)
706 return;
707
708 if (!rm_list->nr)
709 return;
710
711 if (list_empty(head: &msk->conn_list))
712 return;
713
714 for (i = 0; i < rm_list->nr; i++) {
715 u8 rm_id = rm_list->ids[i];
716 bool removed = false;
717
718 mptcp_for_each_subflow_safe(msk, subflow, tmp) {
719 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
720 u8 remote_id = READ_ONCE(subflow->remote_id);
721 int how = RCV_SHUTDOWN | SEND_SHUTDOWN;
722 u8 id = subflow_get_local_id(subflow);
723
724 if ((1 << inet_sk_state_load(sk: ssk)) &
725 (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSING | TCPF_CLOSE))
726 continue;
727 if (rm_type == MPTCP_MIB_RMADDR && remote_id != rm_id)
728 continue;
729 if (rm_type == MPTCP_MIB_RMSUBFLOW && id != rm_id)
730 continue;
731
732 pr_debug(" -> %s rm_list_ids[%d]=%u local_id=%u remote_id=%u mpc_id=%u\n",
733 rm_type == MPTCP_MIB_RMADDR ? "address" : "subflow",
734 i, rm_id, id, remote_id, msk->mpc_endpoint_id);
735 spin_unlock_bh(lock: &msk->pm.lock);
736 mptcp_subflow_shutdown(sk, ssk, how);
737 removed |= subflow->request_join;
738
739 /* the following takes care of updating the subflows counter */
740 mptcp_close_ssk(sk, ssk, subflow);
741 spin_lock_bh(lock: &msk->pm.lock);
742
743 if (rm_type == MPTCP_MIB_RMSUBFLOW)
744 __MPTCP_INC_STATS(net: sock_net(sk), field: rm_type);
745 }
746
747 if (rm_type == MPTCP_MIB_RMADDR) {
748 __MPTCP_INC_STATS(net: sock_net(sk), field: rm_type);
749 if (removed && mptcp_pm_is_kernel(msk))
750 mptcp_pm_nl_rm_addr(msk, rm_id);
751 }
752 }
753}
754
755static void mptcp_pm_rm_addr_recv(struct mptcp_sock *msk)
756{
757 mptcp_pm_rm_addr_or_subflow(msk, rm_list: &msk->pm.rm_list_rx, rm_type: MPTCP_MIB_RMADDR);
758}
759
760void mptcp_pm_rm_subflow(struct mptcp_sock *msk,
761 const struct mptcp_rm_list *rm_list)
762{
763 mptcp_pm_rm_addr_or_subflow(msk, rm_list, rm_type: MPTCP_MIB_RMSUBFLOW);
764}
765
766void mptcp_pm_rm_addr_received(struct mptcp_sock *msk,
767 const struct mptcp_rm_list *rm_list)
768{
769 struct mptcp_pm_data *pm = &msk->pm;
770 u8 i;
771
772 pr_debug("msk=%p remote_ids_nr=%d\n", msk, rm_list->nr);
773
774 for (i = 0; i < rm_list->nr; i++)
775 mptcp_event_addr_removed(msk, id: rm_list->ids[i]);
776
777 spin_lock_bh(lock: &pm->lock);
778 if (mptcp_pm_schedule_work(msk, new_status: MPTCP_PM_RM_ADDR_RECEIVED))
779 pm->rm_list_rx = *rm_list;
780 else
781 __MPTCP_INC_STATS(net: sock_net(sk: (struct sock *)msk), field: MPTCP_MIB_RMADDRDROP);
782 spin_unlock_bh(lock: &pm->lock);
783}
784
785void mptcp_pm_mp_prio_received(struct sock *ssk, u8 bkup)
786{
787 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk: ssk);
788 struct sock *sk = subflow->conn;
789 struct mptcp_sock *msk;
790
791 pr_debug("subflow->backup=%d, bkup=%d\n", subflow->backup, bkup);
792 msk = mptcp_sk(sk);
793 if (subflow->backup != bkup)
794 subflow->backup = bkup;
795
796 mptcp_event(type: MPTCP_EVENT_SUB_PRIORITY, msk, ssk, GFP_ATOMIC);
797}
798
799void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
800{
801 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
802 struct mptcp_sock *msk = mptcp_sk(subflow->conn);
803
804 pr_debug("fail_seq=%llu\n", fail_seq);
805
806 /* After accepting the fail, we can't create any other subflows */
807 spin_lock_bh(lock: &msk->fallback_lock);
808 if (!msk->allow_infinite_fallback) {
809 spin_unlock_bh(lock: &msk->fallback_lock);
810 return;
811 }
812 msk->allow_subflows = false;
813 spin_unlock_bh(lock: &msk->fallback_lock);
814
815 if (!subflow->fail_tout) {
816 pr_debug("send MP_FAIL response and infinite map\n");
817
818 subflow->send_mp_fail = 1;
819 subflow->send_infinite_map = 1;
820 tcp_send_ack(sk);
821 } else {
822 pr_debug("MP_FAIL response received\n");
823 WRITE_ONCE(subflow->fail_tout, 0);
824 }
825}
826
827bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
828 unsigned int opt_size, unsigned int remaining,
829 struct mptcp_addr_info *addr, bool *echo,
830 bool *drop_other_suboptions)
831{
832 int ret = false;
833 u8 add_addr;
834 u8 family;
835 bool port;
836
837 spin_lock_bh(lock: &msk->pm.lock);
838
839 /* double check after the lock is acquired */
840 if (!mptcp_pm_should_add_signal(msk))
841 goto out_unlock;
842
843 /* always drop every other options for pure ack ADD_ADDR; this is a
844 * plain dup-ack from TCP perspective. The other MPTCP-relevant info,
845 * if any, will be carried by the 'original' TCP ack
846 */
847 if (skb && skb_is_tcp_pure_ack(skb)) {
848 remaining += opt_size;
849 *drop_other_suboptions = true;
850 }
851
852 *echo = mptcp_pm_should_add_signal_echo(msk);
853 port = !!(*echo ? msk->pm.remote.port : msk->pm.local.port);
854
855 family = *echo ? msk->pm.remote.family : msk->pm.local.family;
856 if (remaining < mptcp_add_addr_len(family, echo: *echo, port))
857 goto out_unlock;
858
859 if (*echo) {
860 *addr = msk->pm.remote;
861 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO);
862 } else {
863 *addr = msk->pm.local;
864 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL);
865 }
866 WRITE_ONCE(msk->pm.addr_signal, add_addr);
867 ret = true;
868
869out_unlock:
870 spin_unlock_bh(lock: &msk->pm.lock);
871 return ret;
872}
873
874bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
875 struct mptcp_rm_list *rm_list)
876{
877 int ret = false, len;
878 u8 rm_addr;
879
880 spin_lock_bh(lock: &msk->pm.lock);
881
882 /* double check after the lock is acquired */
883 if (!mptcp_pm_should_rm_signal(msk))
884 goto out_unlock;
885
886 rm_addr = msk->pm.addr_signal & ~BIT(MPTCP_RM_ADDR_SIGNAL);
887 len = mptcp_rm_addr_len(rm_list: &msk->pm.rm_list_tx);
888 if (len < 0) {
889 WRITE_ONCE(msk->pm.addr_signal, rm_addr);
890 goto out_unlock;
891 }
892 if (remaining < len)
893 goto out_unlock;
894
895 *rm_list = msk->pm.rm_list_tx;
896 WRITE_ONCE(msk->pm.addr_signal, rm_addr);
897 ret = true;
898
899out_unlock:
900 spin_unlock_bh(lock: &msk->pm.lock);
901 return ret;
902}
903
904int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc)
905{
906 struct mptcp_pm_addr_entry skc_local = { 0 };
907 struct mptcp_addr_info msk_local;
908
909 if (WARN_ON_ONCE(!msk))
910 return -1;
911
912 /* The 0 ID mapping is defined by the first subflow, copied into the msk
913 * addr
914 */
915 mptcp_local_address(skc: (struct sock_common *)msk, addr: &msk_local);
916 mptcp_local_address(skc: (struct sock_common *)skc, addr: &skc_local.addr);
917 if (mptcp_addresses_equal(a: &msk_local, b: &skc_local.addr, use_port: false))
918 return 0;
919
920 skc_local.addr.id = 0;
921 skc_local.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT;
922
923 if (mptcp_pm_is_userspace(msk))
924 return mptcp_userspace_pm_get_local_id(msk, skc: &skc_local);
925 return mptcp_pm_nl_get_local_id(msk, skc: &skc_local);
926}
927
928bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc)
929{
930 struct mptcp_addr_info skc_local;
931
932 mptcp_local_address(skc: (struct sock_common *)skc, addr: &skc_local);
933
934 if (mptcp_pm_is_userspace(msk))
935 return mptcp_userspace_pm_is_backup(msk, skc: &skc_local);
936
937 return mptcp_pm_nl_is_backup(msk, skc: &skc_local);
938}
939
940static void mptcp_pm_subflows_chk_stale(const struct mptcp_sock *msk, struct sock *ssk)
941{
942 struct mptcp_subflow_context *iter, *subflow = mptcp_subflow_ctx(sk: ssk);
943 struct sock *sk = (struct sock *)msk;
944 unsigned int active_max_loss_cnt;
945 struct net *net = sock_net(sk);
946 unsigned int stale_loss_cnt;
947 bool slow;
948
949 stale_loss_cnt = mptcp_stale_loss_cnt(net);
950 if (subflow->stale || !stale_loss_cnt || subflow->stale_count <= stale_loss_cnt)
951 return;
952
953 /* look for another available subflow not in loss state */
954 active_max_loss_cnt = max_t(int, stale_loss_cnt - 1, 1);
955 mptcp_for_each_subflow(msk, iter) {
956 if (iter != subflow && mptcp_subflow_active(subflow: iter) &&
957 iter->stale_count < active_max_loss_cnt) {
958 /* we have some alternatives, try to mark this subflow as idle ...*/
959 slow = lock_sock_fast(sk: ssk);
960 if (!tcp_rtx_and_write_queues_empty(sk: ssk)) {
961 subflow->stale = 1;
962 __mptcp_retransmit_pending_data(sk);
963 MPTCP_INC_STATS(net, field: MPTCP_MIB_SUBFLOWSTALE);
964 }
965 unlock_sock_fast(sk: ssk, slow);
966
967 /* always try to push the pending data regardless of re-injections:
968 * we can possibly use backup subflows now, and subflow selection
969 * is cheap under the msk socket lock
970 */
971 __mptcp_push_pending(sk, flags: 0);
972 return;
973 }
974 }
975}
976
977void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk)
978{
979 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk: ssk);
980 u32 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp);
981
982 /* keep track of rtx periods with no progress */
983 if (!subflow->stale_count) {
984 subflow->stale_rcv_tstamp = rcv_tstamp;
985 subflow->stale_count++;
986 } else if (subflow->stale_rcv_tstamp == rcv_tstamp) {
987 if (subflow->stale_count < U8_MAX)
988 subflow->stale_count++;
989 mptcp_pm_subflows_chk_stale(msk, ssk);
990 } else {
991 subflow->stale_count = 0;
992 mptcp_subflow_set_active(subflow);
993 }
994}
995
996void mptcp_pm_worker(struct mptcp_sock *msk)
997{
998 struct mptcp_pm_data *pm = &msk->pm;
999
1000 msk_owned_by_me(msk);
1001
1002 if (!(pm->status & MPTCP_PM_WORK_MASK))
1003 return;
1004
1005 spin_lock_bh(lock: &msk->pm.lock);
1006
1007 pr_debug("msk=%p status=%x\n", msk, pm->status);
1008 if (pm->status & BIT(MPTCP_PM_ADD_ADDR_SEND_ACK)) {
1009 pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_SEND_ACK);
1010 mptcp_pm_addr_send_ack(msk);
1011 }
1012 if (pm->status & BIT(MPTCP_PM_RM_ADDR_RECEIVED)) {
1013 pm->status &= ~BIT(MPTCP_PM_RM_ADDR_RECEIVED);
1014 mptcp_pm_rm_addr_recv(msk);
1015 }
1016 __mptcp_pm_kernel_worker(msk);
1017
1018 spin_unlock_bh(lock: &msk->pm.lock);
1019}
1020
1021void mptcp_pm_destroy(struct mptcp_sock *msk)
1022{
1023 mptcp_pm_free_anno_list(msk);
1024
1025 if (mptcp_pm_is_userspace(msk))
1026 mptcp_userspace_pm_free_local_addr_list(msk);
1027}
1028
1029void mptcp_pm_data_reset(struct mptcp_sock *msk)
1030{
1031 u8 pm_type = mptcp_get_pm_type(net: sock_net(sk: (struct sock *)msk));
1032 struct mptcp_pm_data *pm = &msk->pm;
1033
1034 memset(&pm->reset, 0, sizeof(pm->reset));
1035 pm->rm_list_tx.nr = 0;
1036 pm->rm_list_rx.nr = 0;
1037 WRITE_ONCE(pm->pm_type, pm_type);
1038
1039 if (pm_type == MPTCP_PM_TYPE_KERNEL) {
1040 bool subflows_allowed = !!mptcp_pm_get_limit_extra_subflows(msk);
1041
1042 /* pm->work_pending must be only be set to 'true' when
1043 * pm->pm_type is set to MPTCP_PM_TYPE_KERNEL
1044 */
1045 WRITE_ONCE(pm->work_pending,
1046 (!!mptcp_pm_get_endp_subflow_max(msk) &&
1047 subflows_allowed) ||
1048 !!mptcp_pm_get_endp_signal_max(msk));
1049 WRITE_ONCE(pm->accept_addr,
1050 !!mptcp_pm_get_limit_add_addr_accepted(msk) &&
1051 subflows_allowed);
1052 WRITE_ONCE(pm->accept_subflow, subflows_allowed);
1053
1054 bitmap_fill(dst: pm->id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
1055 }
1056}
1057
1058void mptcp_pm_data_init(struct mptcp_sock *msk)
1059{
1060 spin_lock_init(&msk->pm.lock);
1061 INIT_LIST_HEAD(list: &msk->pm.anno_list);
1062 INIT_LIST_HEAD(list: &msk->pm.userspace_pm_local_addr_list);
1063 mptcp_pm_data_reset(msk);
1064}
1065
1066void __init mptcp_pm_init(void)
1067{
1068 mptcp_pm_kernel_register();
1069 mptcp_pm_userspace_register();
1070 mptcp_pm_nl_init();
1071}
1072
1073/* Must be called with rcu read lock held */
1074struct mptcp_pm_ops *mptcp_pm_find(const char *name)
1075{
1076 struct mptcp_pm_ops *pm_ops;
1077
1078 list_for_each_entry_rcu(pm_ops, &mptcp_pm_list, list) {
1079 if (!strcmp(pm_ops->name, name))
1080 return pm_ops;
1081 }
1082
1083 return NULL;
1084}
1085
1086int mptcp_pm_validate(struct mptcp_pm_ops *pm_ops)
1087{
1088 return 0;
1089}
1090
1091int mptcp_pm_register(struct mptcp_pm_ops *pm_ops)
1092{
1093 int ret;
1094
1095 ret = mptcp_pm_validate(pm_ops);
1096 if (ret)
1097 return ret;
1098
1099 spin_lock(lock: &mptcp_pm_list_lock);
1100 if (mptcp_pm_find(name: pm_ops->name)) {
1101 spin_unlock(lock: &mptcp_pm_list_lock);
1102 return -EEXIST;
1103 }
1104 list_add_tail_rcu(new: &pm_ops->list, head: &mptcp_pm_list);
1105 spin_unlock(lock: &mptcp_pm_list_lock);
1106
1107 pr_debug("%s registered\n", pm_ops->name);
1108 return 0;
1109}
1110
1111void mptcp_pm_unregister(struct mptcp_pm_ops *pm_ops)
1112{
1113 /* skip unregistering the default path manager */
1114 if (WARN_ON_ONCE(pm_ops == &mptcp_pm_kernel))
1115 return;
1116
1117 spin_lock(lock: &mptcp_pm_list_lock);
1118 list_del_rcu(entry: &pm_ops->list);
1119 spin_unlock(lock: &mptcp_pm_list_lock);
1120}
1121
1122/* Build string with list of available path manager values.
1123 * Similar to tcp_get_available_congestion_control()
1124 */
1125void mptcp_pm_get_available(char *buf, size_t maxlen)
1126{
1127 struct mptcp_pm_ops *pm_ops;
1128 size_t offs = 0;
1129
1130 rcu_read_lock();
1131 list_for_each_entry_rcu(pm_ops, &mptcp_pm_list, list) {
1132 offs += snprintf(buf: buf + offs, size: maxlen - offs, fmt: "%s%s",
1133 offs == 0 ? "" : " ", pm_ops->name);
1134
1135 if (WARN_ON_ONCE(offs >= maxlen))
1136 break;
1137 }
1138 rcu_read_unlock();
1139}
1140

source code of linux/net/mptcp/pm.c