1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* Multipath TCP |
3 | * |
4 | * Copyright (c) 2017 - 2019, Intel Corporation. |
5 | */ |
6 | |
7 | #ifndef __MPTCP_PROTOCOL_H |
8 | #define __MPTCP_PROTOCOL_H |
9 | |
10 | #include <linux/random.h> |
11 | #include <net/tcp.h> |
12 | #include <net/inet_connection_sock.h> |
13 | #include <uapi/linux/mptcp.h> |
14 | #include <net/genetlink.h> |
15 | |
16 | #include "mptcp_pm_gen.h" |
17 | |
18 | #define MPTCP_SUPPORTED_VERSION 1 |
19 | |
20 | /* MPTCP option bits */ |
21 | #define OPTION_MPTCP_MPC_SYN BIT(0) |
22 | #define OPTION_MPTCP_MPC_SYNACK BIT(1) |
23 | #define OPTION_MPTCP_MPC_ACK BIT(2) |
24 | #define OPTION_MPTCP_MPJ_SYN BIT(3) |
25 | #define OPTION_MPTCP_MPJ_SYNACK BIT(4) |
26 | #define OPTION_MPTCP_MPJ_ACK BIT(5) |
27 | #define OPTION_MPTCP_ADD_ADDR BIT(6) |
28 | #define OPTION_MPTCP_RM_ADDR BIT(7) |
29 | #define OPTION_MPTCP_FASTCLOSE BIT(8) |
30 | #define OPTION_MPTCP_PRIO BIT(9) |
31 | #define OPTION_MPTCP_RST BIT(10) |
32 | #define OPTION_MPTCP_DSS BIT(11) |
33 | #define OPTION_MPTCP_FAIL BIT(12) |
34 | |
35 | #define OPTION_MPTCP_CSUMREQD BIT(13) |
36 | |
37 | #define OPTIONS_MPTCP_MPC (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_SYNACK | \ |
38 | OPTION_MPTCP_MPC_ACK) |
39 | #define OPTIONS_MPTCP_MPJ (OPTION_MPTCP_MPJ_SYN | OPTION_MPTCP_MPJ_SYNACK | \ |
40 | OPTION_MPTCP_MPJ_ACK) |
41 | |
42 | /* MPTCP option subtypes */ |
43 | #define MPTCPOPT_MP_CAPABLE 0 |
44 | #define MPTCPOPT_MP_JOIN 1 |
45 | #define MPTCPOPT_DSS 2 |
46 | #define MPTCPOPT_ADD_ADDR 3 |
47 | #define MPTCPOPT_RM_ADDR 4 |
48 | #define MPTCPOPT_MP_PRIO 5 |
49 | #define MPTCPOPT_MP_FAIL 6 |
50 | #define MPTCPOPT_MP_FASTCLOSE 7 |
51 | #define MPTCPOPT_RST 8 |
52 | |
53 | /* MPTCP suboption lengths */ |
54 | #define TCPOLEN_MPTCP_MPC_SYN 4 |
55 | #define TCPOLEN_MPTCP_MPC_SYNACK 12 |
56 | #define TCPOLEN_MPTCP_MPC_ACK 20 |
57 | #define TCPOLEN_MPTCP_MPC_ACK_DATA 22 |
58 | #define TCPOLEN_MPTCP_MPJ_SYN 12 |
59 | #define TCPOLEN_MPTCP_MPJ_SYNACK 16 |
60 | #define TCPOLEN_MPTCP_MPJ_ACK 24 |
61 | #define TCPOLEN_MPTCP_DSS_BASE 4 |
62 | #define TCPOLEN_MPTCP_DSS_ACK32 4 |
63 | #define TCPOLEN_MPTCP_DSS_ACK64 8 |
64 | #define TCPOLEN_MPTCP_DSS_MAP32 10 |
65 | #define TCPOLEN_MPTCP_DSS_MAP64 14 |
66 | #define TCPOLEN_MPTCP_DSS_CHECKSUM 2 |
67 | #define TCPOLEN_MPTCP_ADD_ADDR 16 |
68 | #define TCPOLEN_MPTCP_ADD_ADDR_PORT 18 |
69 | #define TCPOLEN_MPTCP_ADD_ADDR_BASE 8 |
70 | #define TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT 10 |
71 | #define TCPOLEN_MPTCP_ADD_ADDR6 28 |
72 | #define TCPOLEN_MPTCP_ADD_ADDR6_PORT 30 |
73 | #define TCPOLEN_MPTCP_ADD_ADDR6_BASE 20 |
74 | #define TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT 22 |
75 | #define TCPOLEN_MPTCP_PORT_LEN 2 |
76 | #define TCPOLEN_MPTCP_PORT_ALIGN 2 |
77 | #define TCPOLEN_MPTCP_RM_ADDR_BASE 3 |
78 | #define TCPOLEN_MPTCP_PRIO 3 |
79 | #define TCPOLEN_MPTCP_PRIO_ALIGN 4 |
80 | #define TCPOLEN_MPTCP_FASTCLOSE 12 |
81 | #define TCPOLEN_MPTCP_RST 4 |
82 | #define TCPOLEN_MPTCP_FAIL 12 |
83 | |
84 | #define TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM (TCPOLEN_MPTCP_DSS_CHECKSUM + TCPOLEN_MPTCP_MPC_ACK_DATA) |
85 | |
86 | /* MPTCP MP_JOIN flags */ |
87 | #define MPTCPOPT_BACKUP BIT(0) |
88 | #define MPTCPOPT_THMAC_LEN 8 |
89 | |
90 | /* MPTCP MP_CAPABLE flags */ |
91 | #define MPTCP_VERSION_MASK (0x0F) |
92 | #define MPTCP_CAP_CHECKSUM_REQD BIT(7) |
93 | #define MPTCP_CAP_EXTENSIBILITY BIT(6) |
94 | #define MPTCP_CAP_DENY_JOIN_ID0 BIT(5) |
95 | #define MPTCP_CAP_HMAC_SHA256 BIT(0) |
96 | #define MPTCP_CAP_FLAG_MASK (0x1F) |
97 | |
98 | /* MPTCP DSS flags */ |
99 | #define MPTCP_DSS_DATA_FIN BIT(4) |
100 | #define MPTCP_DSS_DSN64 BIT(3) |
101 | #define MPTCP_DSS_HAS_MAP BIT(2) |
102 | #define MPTCP_DSS_ACK64 BIT(1) |
103 | #define MPTCP_DSS_HAS_ACK BIT(0) |
104 | #define MPTCP_DSS_FLAG_MASK (0x1F) |
105 | |
106 | /* MPTCP ADD_ADDR flags */ |
107 | #define MPTCP_ADDR_ECHO BIT(0) |
108 | |
109 | /* MPTCP MP_PRIO flags */ |
110 | #define MPTCP_PRIO_BKUP BIT(0) |
111 | |
112 | /* MPTCP TCPRST flags */ |
113 | #define MPTCP_RST_TRANSIENT BIT(0) |
114 | |
115 | /* MPTCP socket atomic flags */ |
116 | #define MPTCP_NOSPACE 1 |
117 | #define MPTCP_WORK_RTX 2 |
118 | #define MPTCP_FALLBACK_DONE 4 |
119 | #define MPTCP_WORK_CLOSE_SUBFLOW 5 |
120 | |
121 | /* MPTCP socket release cb flags */ |
122 | #define MPTCP_PUSH_PENDING 1 |
123 | #define MPTCP_CLEAN_UNA 2 |
124 | #define MPTCP_ERROR_REPORT 3 |
125 | #define MPTCP_RETRANSMIT 4 |
126 | #define MPTCP_FLUSH_JOIN_LIST 5 |
127 | #define MPTCP_CONNECTED 6 |
128 | #define MPTCP_SYNC_SNDBUF 7 |
129 | |
130 | struct mptcp_skb_cb { |
131 | u64 map_seq; |
132 | u64 end_seq; |
133 | u32 offset; |
134 | u8 has_rxtstamp:1; |
135 | }; |
136 | |
137 | #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) |
138 | |
139 | static inline bool before64(__u64 seq1, __u64 seq2) |
140 | { |
141 | return (__s64)(seq1 - seq2) < 0; |
142 | } |
143 | |
144 | #define after64(seq2, seq1) before64(seq1, seq2) |
145 | |
146 | struct mptcp_options_received { |
147 | u64 sndr_key; |
148 | u64 rcvr_key; |
149 | u64 data_ack; |
150 | u64 data_seq; |
151 | u32 subflow_seq; |
152 | u16 data_len; |
153 | __sum16 csum; |
154 | u16 suboptions; |
155 | u32 token; |
156 | u32 nonce; |
157 | u16 use_map:1, |
158 | dsn64:1, |
159 | data_fin:1, |
160 | use_ack:1, |
161 | ack64:1, |
162 | mpc_map:1, |
163 | reset_reason:4, |
164 | reset_transient:1, |
165 | echo:1, |
166 | backup:1, |
167 | deny_join_id0:1, |
168 | __unused:2; |
169 | u8 join_id; |
170 | u64 thmac; |
171 | u8 hmac[MPTCPOPT_HMAC_LEN]; |
172 | struct mptcp_addr_info addr; |
173 | struct mptcp_rm_list rm_list; |
174 | u64 ahmac; |
175 | u64 fail_seq; |
176 | }; |
177 | |
178 | static inline __be32 mptcp_option(u8 subopt, u8 len, u8 nib, u8 field) |
179 | { |
180 | return htonl((TCPOPT_MPTCP << 24) | (len << 16) | (subopt << 12) | |
181 | ((nib & 0xF) << 8) | field); |
182 | } |
183 | |
184 | enum mptcp_pm_status { |
185 | MPTCP_PM_ADD_ADDR_RECEIVED, |
186 | MPTCP_PM_ADD_ADDR_SEND_ACK, |
187 | MPTCP_PM_RM_ADDR_RECEIVED, |
188 | MPTCP_PM_ESTABLISHED, |
189 | MPTCP_PM_SUBFLOW_ESTABLISHED, |
190 | MPTCP_PM_ALREADY_ESTABLISHED, /* persistent status, set after ESTABLISHED event */ |
191 | MPTCP_PM_MPC_ENDPOINT_ACCOUNTED /* persistent status, set after MPC local address is |
192 | * accounted int id_avail_bitmap |
193 | */ |
194 | }; |
195 | |
196 | enum mptcp_pm_type { |
197 | MPTCP_PM_TYPE_KERNEL = 0, |
198 | MPTCP_PM_TYPE_USERSPACE, |
199 | |
200 | __MPTCP_PM_TYPE_NR, |
201 | __MPTCP_PM_TYPE_MAX = __MPTCP_PM_TYPE_NR - 1, |
202 | }; |
203 | |
204 | /* Status bits below MPTCP_PM_ALREADY_ESTABLISHED need pm worker actions */ |
205 | #define MPTCP_PM_WORK_MASK ((1 << MPTCP_PM_ALREADY_ESTABLISHED) - 1) |
206 | |
207 | enum mptcp_addr_signal_status { |
208 | MPTCP_ADD_ADDR_SIGNAL, |
209 | MPTCP_ADD_ADDR_ECHO, |
210 | MPTCP_RM_ADDR_SIGNAL, |
211 | }; |
212 | |
213 | /* max value of mptcp_addr_info.id */ |
214 | #define MPTCP_PM_MAX_ADDR_ID U8_MAX |
215 | |
216 | struct mptcp_pm_data { |
217 | struct mptcp_addr_info local; |
218 | struct mptcp_addr_info remote; |
219 | struct list_head anno_list; |
220 | struct list_head userspace_pm_local_addr_list; |
221 | |
222 | spinlock_t lock; /*protects the whole PM data */ |
223 | |
224 | u8 addr_signal; |
225 | bool server_side; |
226 | bool work_pending; |
227 | bool accept_addr; |
228 | bool accept_subflow; |
229 | bool remote_deny_join_id0; |
230 | u8 add_addr_signaled; |
231 | u8 add_addr_accepted; |
232 | u8 local_addr_used; |
233 | u8 pm_type; |
234 | u8 subflows; |
235 | u8 status; |
236 | DECLARE_BITMAP(id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); |
237 | struct mptcp_rm_list rm_list_tx; |
238 | struct mptcp_rm_list rm_list_rx; |
239 | }; |
240 | |
241 | struct mptcp_pm_addr_entry { |
242 | struct list_head list; |
243 | struct mptcp_addr_info addr; |
244 | u8 flags; |
245 | int ifindex; |
246 | struct socket *lsk; |
247 | }; |
248 | |
249 | struct mptcp_data_frag { |
250 | struct list_head list; |
251 | u64 data_seq; |
252 | u16 data_len; |
253 | u16 offset; |
254 | u16 overhead; |
255 | u16 already_sent; |
256 | struct page *page; |
257 | }; |
258 | |
259 | /* MPTCP connection sock */ |
260 | struct mptcp_sock { |
261 | /* inet_connection_sock must be the first member */ |
262 | struct inet_connection_sock sk; |
263 | u64 local_key; |
264 | u64 remote_key; |
265 | u64 write_seq; |
266 | u64 bytes_sent; |
267 | u64 snd_nxt; |
268 | u64 bytes_received; |
269 | u64 ack_seq; |
270 | atomic64_t rcv_wnd_sent; |
271 | u64 rcv_data_fin_seq; |
272 | u64 bytes_retrans; |
273 | u64 bytes_consumed; |
274 | int rmem_fwd_alloc; |
275 | int snd_burst; |
276 | int old_wspace; |
277 | u64 recovery_snd_nxt; /* in recovery mode accept up to this seq; |
278 | * recovery related fields are under data_lock |
279 | * protection |
280 | */ |
281 | u64 bytes_acked; |
282 | u64 snd_una; |
283 | u64 wnd_end; |
284 | unsigned long timer_ival; |
285 | u32 token; |
286 | int rmem_released; |
287 | unsigned long flags; |
288 | unsigned long cb_flags; |
289 | unsigned long push_pending; |
290 | bool recovery; /* closing subflow write queue reinjected */ |
291 | bool can_ack; |
292 | bool fully_established; |
293 | bool rcv_data_fin; |
294 | bool snd_data_fin_enable; |
295 | bool rcv_fastclose; |
296 | bool use_64bit_ack; /* Set when we received a 64-bit DSN */ |
297 | bool csum_enabled; |
298 | bool allow_infinite_fallback; |
299 | u8 mpc_endpoint_id; |
300 | u8 recvmsg_inq:1, |
301 | cork:1, |
302 | nodelay:1, |
303 | fastopening:1, |
304 | in_accept_queue:1, |
305 | free_first:1; |
306 | struct work_struct work; |
307 | struct sk_buff *ooo_last_skb; |
308 | struct rb_root out_of_order_queue; |
309 | struct sk_buff_head receive_queue; |
310 | struct list_head conn_list; |
311 | struct list_head rtx_queue; |
312 | struct mptcp_data_frag *first_pending; |
313 | struct list_head join_list; |
314 | struct sock *first; /* The mptcp ops can safely dereference, using suitable |
315 | * ONCE annotation, the subflow outside the socket |
316 | * lock as such sock is freed after close(). |
317 | */ |
318 | struct mptcp_pm_data pm; |
319 | struct mptcp_sched_ops *sched; |
320 | struct { |
321 | u32 space; /* bytes copied in last measurement window */ |
322 | u32 copied; /* bytes copied in this measurement window */ |
323 | u64 time; /* start time of measurement window */ |
324 | u64 rtt_us; /* last maximum rtt of subflows */ |
325 | } rcvq_space; |
326 | u8 scaling_ratio; |
327 | |
328 | u32 subflow_id; |
329 | u32 setsockopt_seq; |
330 | char ca_name[TCP_CA_NAME_MAX]; |
331 | }; |
332 | |
333 | #define mptcp_data_lock(sk) spin_lock_bh(&(sk)->sk_lock.slock) |
334 | #define mptcp_data_unlock(sk) spin_unlock_bh(&(sk)->sk_lock.slock) |
335 | |
336 | #define mptcp_for_each_subflow(__msk, __subflow) \ |
337 | list_for_each_entry(__subflow, &((__msk)->conn_list), node) |
338 | #define mptcp_for_each_subflow_safe(__msk, __subflow, __tmp) \ |
339 | list_for_each_entry_safe(__subflow, __tmp, &((__msk)->conn_list), node) |
340 | |
341 | static inline void msk_owned_by_me(const struct mptcp_sock *msk) |
342 | { |
343 | sock_owned_by_me(sk: (const struct sock *)msk); |
344 | } |
345 | |
346 | #define mptcp_sk(ptr) container_of_const(ptr, struct mptcp_sock, sk.icsk_inet.sk) |
347 | |
348 | /* the msk socket don't use the backlog, also account for the bulk |
349 | * free memory |
350 | */ |
351 | static inline int __mptcp_rmem(const struct sock *sk) |
352 | { |
353 | return atomic_read(v: &sk->sk_rmem_alloc) - READ_ONCE(mptcp_sk(sk)->rmem_released); |
354 | } |
355 | |
356 | static inline int mptcp_win_from_space(const struct sock *sk, int space) |
357 | { |
358 | return __tcp_win_from_space(mptcp_sk(sk)->scaling_ratio, space); |
359 | } |
360 | |
361 | static inline int __mptcp_space(const struct sock *sk) |
362 | { |
363 | return mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - __mptcp_rmem(sk)); |
364 | } |
365 | |
366 | static inline struct mptcp_data_frag *mptcp_send_head(const struct sock *sk) |
367 | { |
368 | const struct mptcp_sock *msk = mptcp_sk(sk); |
369 | |
370 | return READ_ONCE(msk->first_pending); |
371 | } |
372 | |
373 | static inline struct mptcp_data_frag *mptcp_send_next(struct sock *sk) |
374 | { |
375 | struct mptcp_sock *msk = mptcp_sk(sk); |
376 | struct mptcp_data_frag *cur; |
377 | |
378 | cur = msk->first_pending; |
379 | return list_is_last(list: &cur->list, head: &msk->rtx_queue) ? NULL : |
380 | list_next_entry(cur, list); |
381 | } |
382 | |
383 | static inline struct mptcp_data_frag *mptcp_pending_tail(const struct sock *sk) |
384 | { |
385 | const struct mptcp_sock *msk = mptcp_sk(sk); |
386 | |
387 | if (!msk->first_pending) |
388 | return NULL; |
389 | |
390 | if (WARN_ON_ONCE(list_empty(&msk->rtx_queue))) |
391 | return NULL; |
392 | |
393 | return list_last_entry(&msk->rtx_queue, struct mptcp_data_frag, list); |
394 | } |
395 | |
396 | static inline struct mptcp_data_frag *mptcp_rtx_head(struct sock *sk) |
397 | { |
398 | struct mptcp_sock *msk = mptcp_sk(sk); |
399 | |
400 | if (msk->snd_una == READ_ONCE(msk->snd_nxt)) |
401 | return NULL; |
402 | |
403 | return list_first_entry_or_null(&msk->rtx_queue, struct mptcp_data_frag, list); |
404 | } |
405 | |
406 | struct { |
407 | __be64 ; |
408 | __be32 ; |
409 | __be16 ; |
410 | __sum16 ; |
411 | }; |
412 | |
413 | struct mptcp_subflow_request_sock { |
414 | struct tcp_request_sock sk; |
415 | u16 mp_capable : 1, |
416 | mp_join : 1, |
417 | backup : 1, |
418 | csum_reqd : 1, |
419 | allow_join_id0 : 1; |
420 | u8 local_id; |
421 | u8 remote_id; |
422 | u64 local_key; |
423 | u64 idsn; |
424 | u32 token; |
425 | u32 ssn_offset; |
426 | u64 thmac; |
427 | u32 local_nonce; |
428 | u32 remote_nonce; |
429 | struct mptcp_sock *msk; |
430 | struct hlist_nulls_node token_node; |
431 | }; |
432 | |
433 | static inline struct mptcp_subflow_request_sock * |
434 | mptcp_subflow_rsk(const struct request_sock *rsk) |
435 | { |
436 | return (struct mptcp_subflow_request_sock *)rsk; |
437 | } |
438 | |
439 | struct mptcp_delegated_action { |
440 | struct napi_struct napi; |
441 | struct list_head head; |
442 | }; |
443 | |
444 | DECLARE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions); |
445 | |
446 | #define MPTCP_DELEGATE_SCHEDULED 0 |
447 | #define MPTCP_DELEGATE_SEND 1 |
448 | #define MPTCP_DELEGATE_ACK 2 |
449 | #define MPTCP_DELEGATE_SNDBUF 3 |
450 | |
451 | #define MPTCP_DELEGATE_ACTIONS_MASK (~BIT(MPTCP_DELEGATE_SCHEDULED)) |
452 | /* MPTCP subflow context */ |
453 | struct mptcp_subflow_context { |
454 | struct list_head node;/* conn_list of subflows */ |
455 | |
456 | struct_group(reset, |
457 | |
458 | unsigned long avg_pacing_rate; /* protected by msk socket lock */ |
459 | u64 local_key; |
460 | u64 remote_key; |
461 | u64 idsn; |
462 | u64 map_seq; |
463 | u32 snd_isn; |
464 | u32 token; |
465 | u32 rel_write_seq; |
466 | u32 map_subflow_seq; |
467 | u32 ssn_offset; |
468 | u32 map_data_len; |
469 | __wsum map_data_csum; |
470 | u32 map_csum_len; |
471 | u32 request_mptcp : 1, /* send MP_CAPABLE */ |
472 | request_join : 1, /* send MP_JOIN */ |
473 | request_bkup : 1, |
474 | mp_capable : 1, /* remote is MPTCP capable */ |
475 | mp_join : 1, /* remote is JOINing */ |
476 | fully_established : 1, /* path validated */ |
477 | pm_notified : 1, /* PM hook called for established status */ |
478 | conn_finished : 1, |
479 | map_valid : 1, |
480 | map_csum_reqd : 1, |
481 | map_data_fin : 1, |
482 | mpc_map : 1, |
483 | backup : 1, |
484 | send_mp_prio : 1, |
485 | send_mp_fail : 1, |
486 | send_fastclose : 1, |
487 | send_infinite_map : 1, |
488 | remote_key_valid : 1, /* received the peer key from */ |
489 | disposable : 1, /* ctx can be free at ulp release time */ |
490 | stale : 1, /* unable to snd/rcv data, do not use for xmit */ |
491 | local_id_valid : 1, /* local_id is correctly initialized */ |
492 | valid_csum_seen : 1, /* at least one csum validated */ |
493 | is_mptfo : 1, /* subflow is doing TFO */ |
494 | __unused : 9; |
495 | bool data_avail; |
496 | bool scheduled; |
497 | u32 remote_nonce; |
498 | u64 thmac; |
499 | u32 local_nonce; |
500 | u32 remote_token; |
501 | union { |
502 | u8 hmac[MPTCPOPT_HMAC_LEN]; /* MPJ subflow only */ |
503 | u64 iasn; /* initial ack sequence number, MPC subflows only */ |
504 | }; |
505 | u8 local_id; |
506 | u8 remote_id; |
507 | u8 reset_seen:1; |
508 | u8 reset_transient:1; |
509 | u8 reset_reason:4; |
510 | u8 stale_count; |
511 | |
512 | u32 subflow_id; |
513 | |
514 | long delegated_status; |
515 | unsigned long fail_tout; |
516 | |
517 | ); |
518 | |
519 | struct list_head delegated_node; /* link into delegated_action, protected by local BH */ |
520 | |
521 | u32 setsockopt_seq; |
522 | u32 stale_rcv_tstamp; |
523 | int cached_sndbuf; /* sndbuf size when last synced with the msk sndbuf, |
524 | * protected by the msk socket lock |
525 | */ |
526 | |
527 | struct sock *tcp_sock; /* tcp sk backpointer */ |
528 | struct sock *conn; /* parent mptcp_sock */ |
529 | const struct inet_connection_sock_af_ops *icsk_af_ops; |
530 | void (*tcp_state_change)(struct sock *sk); |
531 | void (*tcp_error_report)(struct sock *sk); |
532 | |
533 | struct rcu_head rcu; |
534 | }; |
535 | |
536 | static inline struct mptcp_subflow_context * |
537 | mptcp_subflow_ctx(const struct sock *sk) |
538 | { |
539 | struct inet_connection_sock *icsk = inet_csk(sk); |
540 | |
541 | /* Use RCU on icsk_ulp_data only for sock diag code */ |
542 | return (__force struct mptcp_subflow_context *)icsk->icsk_ulp_data; |
543 | } |
544 | |
545 | static inline struct sock * |
546 | mptcp_subflow_tcp_sock(const struct mptcp_subflow_context *subflow) |
547 | { |
548 | return subflow->tcp_sock; |
549 | } |
550 | |
551 | static inline void |
552 | mptcp_subflow_ctx_reset(struct mptcp_subflow_context *subflow) |
553 | { |
554 | memset(&subflow->reset, 0, sizeof(subflow->reset)); |
555 | subflow->request_mptcp = 1; |
556 | } |
557 | |
558 | static inline u64 |
559 | mptcp_subflow_get_map_offset(const struct mptcp_subflow_context *subflow) |
560 | { |
561 | return tcp_sk(mptcp_subflow_tcp_sock(subflow))->copied_seq - |
562 | subflow->ssn_offset - |
563 | subflow->map_subflow_seq; |
564 | } |
565 | |
566 | static inline u64 |
567 | mptcp_subflow_get_mapped_dsn(const struct mptcp_subflow_context *subflow) |
568 | { |
569 | return subflow->map_seq + mptcp_subflow_get_map_offset(subflow); |
570 | } |
571 | |
572 | void mptcp_subflow_process_delegated(struct sock *ssk, long actions); |
573 | |
574 | static inline void mptcp_subflow_delegate(struct mptcp_subflow_context *subflow, int action) |
575 | { |
576 | long old, set_bits = BIT(MPTCP_DELEGATE_SCHEDULED) | BIT(action); |
577 | struct mptcp_delegated_action *delegated; |
578 | bool schedule; |
579 | |
580 | /* the caller held the subflow bh socket lock */ |
581 | lockdep_assert_in_softirq(); |
582 | |
583 | /* The implied barrier pairs with tcp_release_cb_override() |
584 | * mptcp_napi_poll(), and ensures the below list check sees list |
585 | * updates done prior to delegated status bits changes |
586 | */ |
587 | old = set_mask_bits(&subflow->delegated_status, 0, set_bits); |
588 | if (!(old & BIT(MPTCP_DELEGATE_SCHEDULED))) { |
589 | if (WARN_ON_ONCE(!list_empty(&subflow->delegated_node))) |
590 | return; |
591 | |
592 | delegated = this_cpu_ptr(&mptcp_delegated_actions); |
593 | schedule = list_empty(head: &delegated->head); |
594 | list_add_tail(new: &subflow->delegated_node, head: &delegated->head); |
595 | sock_hold(sk: mptcp_subflow_tcp_sock(subflow)); |
596 | if (schedule) |
597 | napi_schedule(n: &delegated->napi); |
598 | } |
599 | } |
600 | |
601 | static inline struct mptcp_subflow_context * |
602 | mptcp_subflow_delegated_next(struct mptcp_delegated_action *delegated) |
603 | { |
604 | struct mptcp_subflow_context *ret; |
605 | |
606 | if (list_empty(head: &delegated->head)) |
607 | return NULL; |
608 | |
609 | ret = list_first_entry(&delegated->head, struct mptcp_subflow_context, delegated_node); |
610 | list_del_init(entry: &ret->delegated_node); |
611 | return ret; |
612 | } |
613 | |
614 | int mptcp_is_enabled(const struct net *net); |
615 | unsigned int mptcp_get_add_addr_timeout(const struct net *net); |
616 | int mptcp_is_checksum_enabled(const struct net *net); |
617 | int mptcp_allow_join_id0(const struct net *net); |
618 | unsigned int mptcp_stale_loss_cnt(const struct net *net); |
619 | unsigned int mptcp_close_timeout(const struct sock *sk); |
620 | int mptcp_get_pm_type(const struct net *net); |
621 | const char *mptcp_get_scheduler(const struct net *net); |
622 | void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, |
623 | const struct mptcp_options_received *mp_opt); |
624 | bool __mptcp_retransmit_pending_data(struct sock *sk); |
625 | void mptcp_check_and_set_pending(struct sock *sk); |
626 | void __mptcp_push_pending(struct sock *sk, unsigned int flags); |
627 | bool mptcp_subflow_data_available(struct sock *sk); |
628 | void __init mptcp_subflow_init(void); |
629 | void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how); |
630 | void mptcp_close_ssk(struct sock *sk, struct sock *ssk, |
631 | struct mptcp_subflow_context *subflow); |
632 | void __mptcp_subflow_send_ack(struct sock *ssk); |
633 | void mptcp_subflow_reset(struct sock *ssk); |
634 | void mptcp_subflow_queue_clean(struct sock *sk, struct sock *ssk); |
635 | void mptcp_sock_graft(struct sock *sk, struct socket *parent); |
636 | struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk); |
637 | bool __mptcp_close(struct sock *sk, long timeout); |
638 | void mptcp_cancel_work(struct sock *sk); |
639 | void __mptcp_unaccepted_force_close(struct sock *sk); |
640 | void mptcp_set_owner_r(struct sk_buff *skb, struct sock *sk); |
641 | |
642 | bool mptcp_addresses_equal(const struct mptcp_addr_info *a, |
643 | const struct mptcp_addr_info *b, bool use_port); |
644 | void mptcp_local_address(const struct sock_common *skc, struct mptcp_addr_info *addr); |
645 | |
646 | /* called with sk socket lock held */ |
647 | int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, |
648 | const struct mptcp_addr_info *remote); |
649 | int mptcp_subflow_create_socket(struct sock *sk, unsigned short family, |
650 | struct socket **new_sock); |
651 | void mptcp_info2sockaddr(const struct mptcp_addr_info *info, |
652 | struct sockaddr_storage *addr, |
653 | unsigned short family); |
654 | struct mptcp_sched_ops *mptcp_sched_find(const char *name); |
655 | int mptcp_register_scheduler(struct mptcp_sched_ops *sched); |
656 | void mptcp_unregister_scheduler(struct mptcp_sched_ops *sched); |
657 | void mptcp_sched_init(void); |
658 | int mptcp_init_sched(struct mptcp_sock *msk, |
659 | struct mptcp_sched_ops *sched); |
660 | void mptcp_release_sched(struct mptcp_sock *msk); |
661 | void mptcp_subflow_set_scheduled(struct mptcp_subflow_context *subflow, |
662 | bool scheduled); |
663 | struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk); |
664 | struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk); |
665 | int mptcp_sched_get_send(struct mptcp_sock *msk); |
666 | int mptcp_sched_get_retrans(struct mptcp_sock *msk); |
667 | |
668 | static inline u64 mptcp_data_avail(const struct mptcp_sock *msk) |
669 | { |
670 | return READ_ONCE(msk->bytes_received) - READ_ONCE(msk->bytes_consumed); |
671 | } |
672 | |
673 | static inline bool mptcp_epollin_ready(const struct sock *sk) |
674 | { |
675 | /* mptcp doesn't have to deal with small skbs in the receive queue, |
676 | * at it can always coalesce them |
677 | */ |
678 | return (mptcp_data_avail(mptcp_sk(sk)) >= sk->sk_rcvlowat) || |
679 | (mem_cgroup_sockets_enabled && sk->sk_memcg && |
680 | mem_cgroup_under_socket_pressure(memcg: sk->sk_memcg)) || |
681 | READ_ONCE(tcp_memory_pressure); |
682 | } |
683 | |
684 | int mptcp_set_rcvlowat(struct sock *sk, int val); |
685 | |
686 | static inline bool __tcp_can_send(const struct sock *ssk) |
687 | { |
688 | /* only send if our side has not closed yet */ |
689 | return ((1 << inet_sk_state_load(sk: ssk)) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)); |
690 | } |
691 | |
692 | static inline bool __mptcp_subflow_active(struct mptcp_subflow_context *subflow) |
693 | { |
694 | /* can't send if JOIN hasn't completed yet (i.e. is usable for mptcp) */ |
695 | if (subflow->request_join && !subflow->fully_established) |
696 | return false; |
697 | |
698 | return __tcp_can_send(ssk: mptcp_subflow_tcp_sock(subflow)); |
699 | } |
700 | |
701 | void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow); |
702 | |
703 | bool mptcp_subflow_active(struct mptcp_subflow_context *subflow); |
704 | |
705 | void mptcp_subflow_drop_ctx(struct sock *ssk); |
706 | |
707 | static inline void mptcp_subflow_tcp_fallback(struct sock *sk, |
708 | struct mptcp_subflow_context *ctx) |
709 | { |
710 | sk->sk_data_ready = sock_def_readable; |
711 | sk->sk_state_change = ctx->tcp_state_change; |
712 | sk->sk_write_space = sk_stream_write_space; |
713 | sk->sk_error_report = ctx->tcp_error_report; |
714 | |
715 | inet_csk(sk)->icsk_af_ops = ctx->icsk_af_ops; |
716 | } |
717 | |
718 | void __init mptcp_proto_init(void); |
719 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) |
720 | int __init mptcp_proto_v6_init(void); |
721 | #endif |
722 | |
723 | struct sock *mptcp_sk_clone_init(const struct sock *sk, |
724 | const struct mptcp_options_received *mp_opt, |
725 | struct sock *ssk, |
726 | struct request_sock *req); |
727 | void mptcp_get_options(const struct sk_buff *skb, |
728 | struct mptcp_options_received *mp_opt); |
729 | |
730 | void mptcp_finish_connect(struct sock *sk); |
731 | void __mptcp_set_connected(struct sock *sk); |
732 | void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout); |
733 | |
734 | static inline void mptcp_stop_tout_timer(struct sock *sk) |
735 | { |
736 | if (!inet_csk(sk)->icsk_mtup.probe_timestamp) |
737 | return; |
738 | |
739 | sk_stop_timer(sk, timer: &sk->sk_timer); |
740 | inet_csk(sk)->icsk_mtup.probe_timestamp = 0; |
741 | } |
742 | |
743 | static inline void mptcp_set_close_tout(struct sock *sk, unsigned long tout) |
744 | { |
745 | /* avoid 0 timestamp, as that means no close timeout */ |
746 | inet_csk(sk)->icsk_mtup.probe_timestamp = tout ? : 1; |
747 | } |
748 | |
749 | static inline void mptcp_start_tout_timer(struct sock *sk) |
750 | { |
751 | mptcp_set_close_tout(sk, tcp_jiffies32); |
752 | mptcp_reset_tout_timer(mptcp_sk(sk), fail_tout: 0); |
753 | } |
754 | |
755 | static inline bool mptcp_is_fully_established(struct sock *sk) |
756 | { |
757 | return inet_sk_state_load(sk) == TCP_ESTABLISHED && |
758 | READ_ONCE(mptcp_sk(sk)->fully_established); |
759 | } |
760 | |
761 | void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk); |
762 | void mptcp_data_ready(struct sock *sk, struct sock *ssk); |
763 | bool mptcp_finish_join(struct sock *sk); |
764 | bool mptcp_schedule_work(struct sock *sk); |
765 | int mptcp_setsockopt(struct sock *sk, int level, int optname, |
766 | sockptr_t optval, unsigned int optlen); |
767 | int mptcp_getsockopt(struct sock *sk, int level, int optname, |
768 | char __user *optval, int __user *option); |
769 | |
770 | u64 __mptcp_expand_seq(u64 old_seq, u64 cur_seq); |
771 | static inline u64 mptcp_expand_seq(u64 old_seq, u64 cur_seq, bool use_64bit) |
772 | { |
773 | if (use_64bit) |
774 | return cur_seq; |
775 | |
776 | return __mptcp_expand_seq(old_seq, cur_seq); |
777 | } |
778 | void __mptcp_check_push(struct sock *sk, struct sock *ssk); |
779 | void __mptcp_data_acked(struct sock *sk); |
780 | void __mptcp_error_report(struct sock *sk); |
781 | bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit); |
782 | static inline bool mptcp_data_fin_enabled(const struct mptcp_sock *msk) |
783 | { |
784 | return READ_ONCE(msk->snd_data_fin_enable) && |
785 | READ_ONCE(msk->write_seq) == READ_ONCE(msk->snd_nxt); |
786 | } |
787 | |
788 | static inline void __mptcp_sync_sndbuf(struct sock *sk) |
789 | { |
790 | struct mptcp_subflow_context *subflow; |
791 | int ssk_sndbuf, new_sndbuf; |
792 | |
793 | if (sk->sk_userlocks & SOCK_SNDBUF_LOCK) |
794 | return; |
795 | |
796 | new_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[0]; |
797 | mptcp_for_each_subflow(mptcp_sk(sk), subflow) { |
798 | ssk_sndbuf = READ_ONCE(mptcp_subflow_tcp_sock(subflow)->sk_sndbuf); |
799 | |
800 | subflow->cached_sndbuf = ssk_sndbuf; |
801 | new_sndbuf += ssk_sndbuf; |
802 | } |
803 | |
804 | /* the msk max wmem limit is <nr_subflows> * tcp wmem[2] */ |
805 | WRITE_ONCE(sk->sk_sndbuf, new_sndbuf); |
806 | } |
807 | |
808 | /* The called held both the msk socket and the subflow socket locks, |
809 | * possibly under BH |
810 | */ |
811 | static inline void __mptcp_propagate_sndbuf(struct sock *sk, struct sock *ssk) |
812 | { |
813 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk: ssk); |
814 | |
815 | if (READ_ONCE(ssk->sk_sndbuf) != subflow->cached_sndbuf) |
816 | __mptcp_sync_sndbuf(sk); |
817 | } |
818 | |
819 | /* the caller held only the subflow socket lock, either in process or |
820 | * BH context. Additionally this can be called under the msk data lock, |
821 | * so we can't acquire such lock here: let the delegate action acquires |
822 | * the needed locks in suitable order. |
823 | */ |
824 | static inline void mptcp_propagate_sndbuf(struct sock *sk, struct sock *ssk) |
825 | { |
826 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk: ssk); |
827 | |
828 | if (likely(READ_ONCE(ssk->sk_sndbuf) == subflow->cached_sndbuf)) |
829 | return; |
830 | |
831 | local_bh_disable(); |
832 | mptcp_subflow_delegate(subflow, MPTCP_DELEGATE_SNDBUF); |
833 | local_bh_enable(); |
834 | } |
835 | |
836 | static inline void mptcp_write_space(struct sock *sk) |
837 | { |
838 | if (sk_stream_is_writeable(sk)) { |
839 | /* pairs with memory barrier in mptcp_poll */ |
840 | smp_mb(); |
841 | if (test_and_clear_bit(MPTCP_NOSPACE, addr: &mptcp_sk(sk)->flags)) |
842 | sk_stream_write_space(sk); |
843 | } |
844 | } |
845 | |
846 | void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags); |
847 | |
848 | #define MPTCP_TOKEN_MAX_RETRIES 4 |
849 | |
850 | void __init mptcp_token_init(void); |
851 | static inline void mptcp_token_init_request(struct request_sock *req) |
852 | { |
853 | mptcp_subflow_rsk(rsk: req)->token_node.pprev = NULL; |
854 | } |
855 | |
856 | int mptcp_token_new_request(struct request_sock *req); |
857 | void mptcp_token_destroy_request(struct request_sock *req); |
858 | int mptcp_token_new_connect(struct sock *ssk); |
859 | void mptcp_token_accept(struct mptcp_subflow_request_sock *r, |
860 | struct mptcp_sock *msk); |
861 | bool mptcp_token_exists(u32 token); |
862 | struct mptcp_sock *mptcp_token_get_sock(struct net *net, u32 token); |
863 | struct mptcp_sock *mptcp_token_iter_next(const struct net *net, long *s_slot, |
864 | long *s_num); |
865 | void mptcp_token_destroy(struct mptcp_sock *msk); |
866 | |
867 | void mptcp_crypto_key_sha(u64 key, u32 *token, u64 *idsn); |
868 | |
869 | void mptcp_crypto_hmac_sha(u64 key1, u64 key2, u8 *msg, int len, void *hmac); |
870 | __sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum); |
871 | |
872 | void __init mptcp_pm_init(void); |
873 | void mptcp_pm_data_init(struct mptcp_sock *msk); |
874 | void mptcp_pm_data_reset(struct mptcp_sock *msk); |
875 | int mptcp_pm_parse_addr(struct nlattr *attr, struct genl_info *info, |
876 | struct mptcp_addr_info *addr); |
877 | int mptcp_pm_parse_entry(struct nlattr *attr, struct genl_info *info, |
878 | bool require_family, |
879 | struct mptcp_pm_addr_entry *entry); |
880 | bool mptcp_pm_addr_families_match(const struct sock *sk, |
881 | const struct mptcp_addr_info *loc, |
882 | const struct mptcp_addr_info *rem); |
883 | void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk); |
884 | void mptcp_pm_nl_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk); |
885 | void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side); |
886 | void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk); |
887 | bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk); |
888 | void mptcp_pm_connection_closed(struct mptcp_sock *msk); |
889 | void mptcp_pm_subflow_established(struct mptcp_sock *msk); |
890 | bool mptcp_pm_nl_check_work_pending(struct mptcp_sock *msk); |
891 | void mptcp_pm_subflow_check_next(struct mptcp_sock *msk, |
892 | const struct mptcp_subflow_context *subflow); |
893 | void mptcp_pm_add_addr_received(const struct sock *ssk, |
894 | const struct mptcp_addr_info *addr); |
895 | void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk, |
896 | const struct mptcp_addr_info *addr); |
897 | void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk); |
898 | void mptcp_pm_nl_addr_send_ack(struct mptcp_sock *msk); |
899 | void mptcp_pm_rm_addr_received(struct mptcp_sock *msk, |
900 | const struct mptcp_rm_list *rm_list); |
901 | void mptcp_pm_mp_prio_received(struct sock *sk, u8 bkup); |
902 | void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq); |
903 | int mptcp_pm_nl_mp_prio_send_ack(struct mptcp_sock *msk, |
904 | struct mptcp_addr_info *addr, |
905 | struct mptcp_addr_info *rem, |
906 | u8 bkup); |
907 | bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, |
908 | const struct mptcp_addr_info *addr); |
909 | void mptcp_pm_free_anno_list(struct mptcp_sock *msk); |
910 | bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk); |
911 | struct mptcp_pm_add_entry * |
912 | mptcp_pm_del_add_timer(struct mptcp_sock *msk, |
913 | const struct mptcp_addr_info *addr, bool check_id); |
914 | struct mptcp_pm_add_entry * |
915 | mptcp_lookup_anno_list_by_saddr(const struct mptcp_sock *msk, |
916 | const struct mptcp_addr_info *addr); |
917 | int mptcp_pm_get_flags_and_ifindex_by_id(struct mptcp_sock *msk, |
918 | unsigned int id, |
919 | u8 *flags, int *ifindex); |
920 | int mptcp_pm_nl_get_flags_and_ifindex_by_id(struct mptcp_sock *msk, unsigned int id, |
921 | u8 *flags, int *ifindex); |
922 | int mptcp_userspace_pm_get_flags_and_ifindex_by_id(struct mptcp_sock *msk, |
923 | unsigned int id, |
924 | u8 *flags, int *ifindex); |
925 | int mptcp_pm_set_flags(struct net *net, struct nlattr *token, |
926 | struct mptcp_pm_addr_entry *loc, |
927 | struct mptcp_pm_addr_entry *rem, u8 bkup); |
928 | int mptcp_pm_nl_set_flags(struct net *net, struct mptcp_pm_addr_entry *addr, u8 bkup); |
929 | int mptcp_userspace_pm_set_flags(struct net *net, struct nlattr *token, |
930 | struct mptcp_pm_addr_entry *loc, |
931 | struct mptcp_pm_addr_entry *rem, u8 bkup); |
932 | int mptcp_pm_announce_addr(struct mptcp_sock *msk, |
933 | const struct mptcp_addr_info *addr, |
934 | bool echo); |
935 | int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list); |
936 | int mptcp_pm_remove_subflow(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list); |
937 | void mptcp_pm_remove_addrs(struct mptcp_sock *msk, struct list_head *rm_list); |
938 | void mptcp_pm_remove_addrs_and_subflows(struct mptcp_sock *msk, |
939 | struct list_head *rm_list); |
940 | |
941 | void mptcp_free_local_addr_list(struct mptcp_sock *msk); |
942 | |
943 | void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk, |
944 | const struct sock *ssk, gfp_t gfp); |
945 | void mptcp_event_addr_announced(const struct sock *ssk, const struct mptcp_addr_info *info); |
946 | void mptcp_event_addr_removed(const struct mptcp_sock *msk, u8 id); |
947 | void mptcp_event_pm_listener(const struct sock *ssk, |
948 | enum mptcp_event_type event); |
949 | bool mptcp_userspace_pm_active(const struct mptcp_sock *msk); |
950 | |
951 | void mptcp_fastopen_gen_msk_ackseq(struct mptcp_sock *msk, struct mptcp_subflow_context *subflow, |
952 | const struct mptcp_options_received *mp_opt); |
953 | void mptcp_fastopen_subflow_synack_set_params(struct mptcp_subflow_context *subflow, |
954 | struct request_sock *req); |
955 | |
956 | static inline bool mptcp_pm_should_add_signal(struct mptcp_sock *msk) |
957 | { |
958 | return READ_ONCE(msk->pm.addr_signal) & |
959 | (BIT(MPTCP_ADD_ADDR_SIGNAL) | BIT(MPTCP_ADD_ADDR_ECHO)); |
960 | } |
961 | |
962 | static inline bool mptcp_pm_should_add_signal_addr(struct mptcp_sock *msk) |
963 | { |
964 | return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_ADD_ADDR_SIGNAL); |
965 | } |
966 | |
967 | static inline bool mptcp_pm_should_add_signal_echo(struct mptcp_sock *msk) |
968 | { |
969 | return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_ADD_ADDR_ECHO); |
970 | } |
971 | |
972 | static inline bool mptcp_pm_should_rm_signal(struct mptcp_sock *msk) |
973 | { |
974 | return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_RM_ADDR_SIGNAL); |
975 | } |
976 | |
977 | static inline bool mptcp_pm_is_userspace(const struct mptcp_sock *msk) |
978 | { |
979 | return READ_ONCE(msk->pm.pm_type) == MPTCP_PM_TYPE_USERSPACE; |
980 | } |
981 | |
982 | static inline bool mptcp_pm_is_kernel(const struct mptcp_sock *msk) |
983 | { |
984 | return READ_ONCE(msk->pm.pm_type) == MPTCP_PM_TYPE_KERNEL; |
985 | } |
986 | |
987 | static inline unsigned int mptcp_add_addr_len(int family, bool echo, bool port) |
988 | { |
989 | u8 len = TCPOLEN_MPTCP_ADD_ADDR_BASE; |
990 | |
991 | if (family == AF_INET6) |
992 | len = TCPOLEN_MPTCP_ADD_ADDR6_BASE; |
993 | if (!echo) |
994 | len += MPTCPOPT_THMAC_LEN; |
995 | /* account for 2 trailing 'nop' options */ |
996 | if (port) |
997 | len += TCPOLEN_MPTCP_PORT_LEN + TCPOLEN_MPTCP_PORT_ALIGN; |
998 | |
999 | return len; |
1000 | } |
1001 | |
1002 | static inline int mptcp_rm_addr_len(const struct mptcp_rm_list *rm_list) |
1003 | { |
1004 | if (rm_list->nr == 0 || rm_list->nr > MPTCP_RM_IDS_MAX) |
1005 | return -EINVAL; |
1006 | |
1007 | return TCPOLEN_MPTCP_RM_ADDR_BASE + roundup(rm_list->nr - 1, 4) + 1; |
1008 | } |
1009 | |
1010 | bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb, |
1011 | unsigned int opt_size, unsigned int remaining, |
1012 | struct mptcp_addr_info *addr, bool *echo, |
1013 | bool *drop_other_suboptions); |
1014 | bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining, |
1015 | struct mptcp_rm_list *rm_list); |
1016 | int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc); |
1017 | int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc); |
1018 | int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc); |
1019 | |
1020 | void __init mptcp_pm_nl_init(void); |
1021 | void mptcp_pm_nl_work(struct mptcp_sock *msk); |
1022 | void mptcp_pm_nl_rm_subflow_received(struct mptcp_sock *msk, |
1023 | const struct mptcp_rm_list *rm_list); |
1024 | unsigned int mptcp_pm_get_add_addr_signal_max(const struct mptcp_sock *msk); |
1025 | unsigned int mptcp_pm_get_add_addr_accept_max(const struct mptcp_sock *msk); |
1026 | unsigned int mptcp_pm_get_subflows_max(const struct mptcp_sock *msk); |
1027 | unsigned int mptcp_pm_get_local_addr_max(const struct mptcp_sock *msk); |
1028 | |
1029 | /* called under PM lock */ |
1030 | static inline void __mptcp_pm_close_subflow(struct mptcp_sock *msk) |
1031 | { |
1032 | if (--msk->pm.subflows < mptcp_pm_get_subflows_max(msk)) |
1033 | WRITE_ONCE(msk->pm.accept_subflow, true); |
1034 | } |
1035 | |
1036 | static inline void mptcp_pm_close_subflow(struct mptcp_sock *msk) |
1037 | { |
1038 | spin_lock_bh(lock: &msk->pm.lock); |
1039 | __mptcp_pm_close_subflow(msk); |
1040 | spin_unlock_bh(lock: &msk->pm.lock); |
1041 | } |
1042 | |
1043 | void mptcp_sockopt_sync(struct mptcp_sock *msk, struct sock *ssk); |
1044 | void mptcp_sockopt_sync_locked(struct mptcp_sock *msk, struct sock *ssk); |
1045 | |
1046 | static inline struct mptcp_ext *mptcp_get_ext(const struct sk_buff *skb) |
1047 | { |
1048 | return (struct mptcp_ext *)skb_ext_find(skb, id: SKB_EXT_MPTCP); |
1049 | } |
1050 | |
1051 | void mptcp_diag_subflow_init(struct tcp_ulp_ops *ops); |
1052 | |
1053 | static inline bool __mptcp_check_fallback(const struct mptcp_sock *msk) |
1054 | { |
1055 | return test_bit(MPTCP_FALLBACK_DONE, &msk->flags); |
1056 | } |
1057 | |
1058 | static inline bool mptcp_check_fallback(const struct sock *sk) |
1059 | { |
1060 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); |
1061 | struct mptcp_sock *msk = mptcp_sk(subflow->conn); |
1062 | |
1063 | return __mptcp_check_fallback(msk); |
1064 | } |
1065 | |
1066 | static inline void __mptcp_do_fallback(struct mptcp_sock *msk) |
1067 | { |
1068 | if (__mptcp_check_fallback(msk)) { |
1069 | pr_debug("TCP fallback already done (msk=%p)" , msk); |
1070 | return; |
1071 | } |
1072 | set_bit(MPTCP_FALLBACK_DONE, addr: &msk->flags); |
1073 | } |
1074 | |
1075 | static inline void mptcp_do_fallback(struct sock *ssk) |
1076 | { |
1077 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk: ssk); |
1078 | struct sock *sk = subflow->conn; |
1079 | struct mptcp_sock *msk; |
1080 | |
1081 | msk = mptcp_sk(sk); |
1082 | __mptcp_do_fallback(msk); |
1083 | if (READ_ONCE(msk->snd_data_fin_enable) && !(ssk->sk_shutdown & SEND_SHUTDOWN)) { |
1084 | gfp_t saved_allocation = ssk->sk_allocation; |
1085 | |
1086 | /* we are in a atomic (BH) scope, override ssk default for data |
1087 | * fin allocation |
1088 | */ |
1089 | ssk->sk_allocation = GFP_ATOMIC; |
1090 | ssk->sk_shutdown |= SEND_SHUTDOWN; |
1091 | tcp_shutdown(sk: ssk, SEND_SHUTDOWN); |
1092 | ssk->sk_allocation = saved_allocation; |
1093 | } |
1094 | } |
1095 | |
1096 | #define pr_fallback(a) pr_debug("%s:fallback to TCP (msk=%p)", __func__, a) |
1097 | |
1098 | static inline bool mptcp_check_infinite_map(struct sk_buff *skb) |
1099 | { |
1100 | struct mptcp_ext *mpext; |
1101 | |
1102 | mpext = skb ? mptcp_get_ext(skb) : NULL; |
1103 | if (mpext && mpext->infinite_map) |
1104 | return true; |
1105 | |
1106 | return false; |
1107 | } |
1108 | |
1109 | static inline bool is_active_ssk(struct mptcp_subflow_context *subflow) |
1110 | { |
1111 | return (subflow->request_mptcp || subflow->request_join); |
1112 | } |
1113 | |
1114 | static inline bool subflow_simultaneous_connect(struct sock *sk) |
1115 | { |
1116 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); |
1117 | |
1118 | return sk->sk_state == TCP_ESTABLISHED && |
1119 | is_active_ssk(subflow) && |
1120 | !subflow->conn_finished; |
1121 | } |
1122 | |
1123 | #ifdef CONFIG_SYN_COOKIES |
1124 | void subflow_init_req_cookie_join_save(const struct mptcp_subflow_request_sock *subflow_req, |
1125 | struct sk_buff *skb); |
1126 | bool mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subflow_req, |
1127 | struct sk_buff *skb); |
1128 | void __init mptcp_join_cookie_init(void); |
1129 | #else |
1130 | static inline void |
1131 | subflow_init_req_cookie_join_save(const struct mptcp_subflow_request_sock *subflow_req, |
1132 | struct sk_buff *skb) {} |
1133 | static inline bool |
1134 | mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subflow_req, |
1135 | struct sk_buff *skb) |
1136 | { |
1137 | return false; |
1138 | } |
1139 | |
1140 | static inline void mptcp_join_cookie_init(void) {} |
1141 | #endif |
1142 | |
1143 | #endif /* __MPTCP_PROTOCOL_H */ |
1144 | |