1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Multipath TCP |
3 | * |
4 | * Copyright (c) 2022, SUSE. |
5 | */ |
6 | |
7 | #define pr_fmt(fmt) "MPTCP: " fmt |
8 | |
9 | #include <linux/kernel.h> |
10 | #include <linux/module.h> |
11 | #include <linux/list.h> |
12 | #include <linux/rculist.h> |
13 | #include <linux/spinlock.h> |
14 | #include "protocol.h" |
15 | |
16 | static DEFINE_SPINLOCK(mptcp_sched_list_lock); |
17 | static LIST_HEAD(mptcp_sched_list); |
18 | |
19 | static int mptcp_sched_default_get_subflow(struct mptcp_sock *msk, |
20 | struct mptcp_sched_data *data) |
21 | { |
22 | struct sock *ssk; |
23 | |
24 | ssk = data->reinject ? mptcp_subflow_get_retrans(msk) : |
25 | mptcp_subflow_get_send(msk); |
26 | if (!ssk) |
27 | return -EINVAL; |
28 | |
29 | mptcp_subflow_set_scheduled(subflow: mptcp_subflow_ctx(sk: ssk), scheduled: true); |
30 | return 0; |
31 | } |
32 | |
33 | static struct mptcp_sched_ops mptcp_sched_default = { |
34 | .get_subflow = mptcp_sched_default_get_subflow, |
35 | .name = "default" , |
36 | .owner = THIS_MODULE, |
37 | }; |
38 | |
39 | /* Must be called with rcu read lock held */ |
40 | struct mptcp_sched_ops *mptcp_sched_find(const char *name) |
41 | { |
42 | struct mptcp_sched_ops *sched, *ret = NULL; |
43 | |
44 | list_for_each_entry_rcu(sched, &mptcp_sched_list, list) { |
45 | if (!strcmp(sched->name, name)) { |
46 | ret = sched; |
47 | break; |
48 | } |
49 | } |
50 | |
51 | return ret; |
52 | } |
53 | |
54 | int mptcp_register_scheduler(struct mptcp_sched_ops *sched) |
55 | { |
56 | if (!sched->get_subflow) |
57 | return -EINVAL; |
58 | |
59 | spin_lock(lock: &mptcp_sched_list_lock); |
60 | if (mptcp_sched_find(name: sched->name)) { |
61 | spin_unlock(lock: &mptcp_sched_list_lock); |
62 | return -EEXIST; |
63 | } |
64 | list_add_tail_rcu(new: &sched->list, head: &mptcp_sched_list); |
65 | spin_unlock(lock: &mptcp_sched_list_lock); |
66 | |
67 | pr_debug("%s registered" , sched->name); |
68 | return 0; |
69 | } |
70 | |
71 | void mptcp_unregister_scheduler(struct mptcp_sched_ops *sched) |
72 | { |
73 | if (sched == &mptcp_sched_default) |
74 | return; |
75 | |
76 | spin_lock(lock: &mptcp_sched_list_lock); |
77 | list_del_rcu(entry: &sched->list); |
78 | spin_unlock(lock: &mptcp_sched_list_lock); |
79 | } |
80 | |
81 | void mptcp_sched_init(void) |
82 | { |
83 | mptcp_register_scheduler(sched: &mptcp_sched_default); |
84 | } |
85 | |
86 | int mptcp_init_sched(struct mptcp_sock *msk, |
87 | struct mptcp_sched_ops *sched) |
88 | { |
89 | if (!sched) |
90 | sched = &mptcp_sched_default; |
91 | |
92 | if (!bpf_try_module_get(data: sched, owner: sched->owner)) |
93 | return -EBUSY; |
94 | |
95 | msk->sched = sched; |
96 | if (msk->sched->init) |
97 | msk->sched->init(msk); |
98 | |
99 | pr_debug("sched=%s" , msk->sched->name); |
100 | |
101 | return 0; |
102 | } |
103 | |
104 | void mptcp_release_sched(struct mptcp_sock *msk) |
105 | { |
106 | struct mptcp_sched_ops *sched = msk->sched; |
107 | |
108 | if (!sched) |
109 | return; |
110 | |
111 | msk->sched = NULL; |
112 | if (sched->release) |
113 | sched->release(msk); |
114 | |
115 | bpf_module_put(data: sched, owner: sched->owner); |
116 | } |
117 | |
118 | void mptcp_subflow_set_scheduled(struct mptcp_subflow_context *subflow, |
119 | bool scheduled) |
120 | { |
121 | WRITE_ONCE(subflow->scheduled, scheduled); |
122 | } |
123 | |
124 | int mptcp_sched_get_send(struct mptcp_sock *msk) |
125 | { |
126 | struct mptcp_subflow_context *subflow; |
127 | struct mptcp_sched_data data; |
128 | |
129 | msk_owned_by_me(msk); |
130 | |
131 | /* the following check is moved out of mptcp_subflow_get_send */ |
132 | if (__mptcp_check_fallback(msk)) { |
133 | if (msk->first && |
134 | __tcp_can_send(ssk: msk->first) && |
135 | sk_stream_memory_free(sk: msk->first)) { |
136 | mptcp_subflow_set_scheduled(subflow: mptcp_subflow_ctx(sk: msk->first), scheduled: true); |
137 | return 0; |
138 | } |
139 | return -EINVAL; |
140 | } |
141 | |
142 | mptcp_for_each_subflow(msk, subflow) { |
143 | if (READ_ONCE(subflow->scheduled)) |
144 | return 0; |
145 | } |
146 | |
147 | data.reinject = false; |
148 | if (msk->sched == &mptcp_sched_default || !msk->sched) |
149 | return mptcp_sched_default_get_subflow(msk, data: &data); |
150 | return msk->sched->get_subflow(msk, &data); |
151 | } |
152 | |
153 | int mptcp_sched_get_retrans(struct mptcp_sock *msk) |
154 | { |
155 | struct mptcp_subflow_context *subflow; |
156 | struct mptcp_sched_data data; |
157 | |
158 | msk_owned_by_me(msk); |
159 | |
160 | /* the following check is moved out of mptcp_subflow_get_retrans */ |
161 | if (__mptcp_check_fallback(msk)) |
162 | return -EINVAL; |
163 | |
164 | mptcp_for_each_subflow(msk, subflow) { |
165 | if (READ_ONCE(subflow->scheduled)) |
166 | return 0; |
167 | } |
168 | |
169 | data.reinject = true; |
170 | if (msk->sched == &mptcp_sched_default || !msk->sched) |
171 | return mptcp_sched_default_get_subflow(msk, data: &data); |
172 | return msk->sched->get_subflow(msk, &data); |
173 | } |
174 | |