1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE |
4 | * |
5 | * Definitions for the SMC module (socket related) |
6 | * |
7 | * Copyright IBM Corp. 2016 |
8 | * |
9 | * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> |
10 | */ |
11 | #ifndef __SMC_H |
12 | #define __SMC_H |
13 | |
14 | #include <linux/socket.h> |
15 | #include <linux/types.h> |
16 | #include <linux/compiler.h> /* __aligned */ |
17 | #include <net/genetlink.h> |
18 | #include <net/sock.h> |
19 | |
20 | #include "smc_ib.h" |
21 | |
22 | #define SMC_V1 1 /* SMC version V1 */ |
23 | #define SMC_V2 2 /* SMC version V2 */ |
24 | |
25 | #define SMC_RELEASE_0 0 |
26 | #define SMC_RELEASE_1 1 |
27 | #define SMC_RELEASE SMC_RELEASE_1 /* the latest release version */ |
28 | |
29 | #define SMCPROTO_SMC 0 /* SMC protocol, IPv4 */ |
30 | #define SMCPROTO_SMC6 1 /* SMC protocol, IPv6 */ |
31 | |
32 | #define SMC_AUTOCORKING_DEFAULT_SIZE 0x10000 /* 64K by default */ |
33 | |
34 | extern struct proto smc_proto; |
35 | extern struct proto smc_proto6; |
36 | |
37 | #ifdef ATOMIC64_INIT |
38 | #define KERNEL_HAS_ATOMIC64 |
39 | #endif |
40 | |
41 | enum smc_state { /* possible states of an SMC socket */ |
42 | SMC_ACTIVE = 1, |
43 | SMC_INIT = 2, |
44 | SMC_CLOSED = 7, |
45 | SMC_LISTEN = 10, |
46 | /* normal close */ |
47 | SMC_PEERCLOSEWAIT1 = 20, |
48 | SMC_PEERCLOSEWAIT2 = 21, |
49 | SMC_APPFINCLOSEWAIT = 24, |
50 | SMC_APPCLOSEWAIT1 = 22, |
51 | SMC_APPCLOSEWAIT2 = 23, |
52 | SMC_PEERFINCLOSEWAIT = 25, |
53 | /* abnormal close */ |
54 | SMC_PEERABORTWAIT = 26, |
55 | SMC_PROCESSABORT = 27, |
56 | }; |
57 | |
58 | enum smc_supplemental_features { |
59 | SMC_SPF_EMULATED_ISM_DEV = 0, |
60 | }; |
61 | |
62 | #define SMC_FEATURE_MASK \ |
63 | (BIT(SMC_SPF_EMULATED_ISM_DEV)) |
64 | |
65 | struct smc_link_group; |
66 | |
67 | struct smc_wr_rx_hdr { /* common prefix part of LLC and CDC to demultiplex */ |
68 | union { |
69 | u8 type; |
70 | #if defined(__BIG_ENDIAN_BITFIELD) |
71 | struct { |
72 | u8 llc_version:4, |
73 | llc_type:4; |
74 | }; |
75 | #elif defined(__LITTLE_ENDIAN_BITFIELD) |
76 | struct { |
77 | u8 llc_type:4, |
78 | llc_version:4; |
79 | }; |
80 | #endif |
81 | }; |
82 | } __aligned(1); |
83 | |
84 | struct smc_cdc_conn_state_flags { |
85 | #if defined(__BIG_ENDIAN_BITFIELD) |
86 | u8 peer_done_writing : 1; /* Sending done indicator */ |
87 | u8 peer_conn_closed : 1; /* Peer connection closed indicator */ |
88 | u8 peer_conn_abort : 1; /* Abnormal close indicator */ |
89 | u8 reserved : 5; |
90 | #elif defined(__LITTLE_ENDIAN_BITFIELD) |
91 | u8 reserved : 5; |
92 | u8 peer_conn_abort : 1; |
93 | u8 peer_conn_closed : 1; |
94 | u8 peer_done_writing : 1; |
95 | #endif |
96 | }; |
97 | |
98 | struct smc_cdc_producer_flags { |
99 | #if defined(__BIG_ENDIAN_BITFIELD) |
100 | u8 write_blocked : 1; /* Writing Blocked, no rx buf space */ |
101 | u8 urg_data_pending : 1; /* Urgent Data Pending */ |
102 | u8 urg_data_present : 1; /* Urgent Data Present */ |
103 | u8 cons_curs_upd_req : 1; /* cursor update requested */ |
104 | u8 failover_validation : 1;/* message replay due to failover */ |
105 | u8 reserved : 3; |
106 | #elif defined(__LITTLE_ENDIAN_BITFIELD) |
107 | u8 reserved : 3; |
108 | u8 failover_validation : 1; |
109 | u8 cons_curs_upd_req : 1; |
110 | u8 urg_data_present : 1; |
111 | u8 urg_data_pending : 1; |
112 | u8 write_blocked : 1; |
113 | #endif |
114 | }; |
115 | |
116 | /* in host byte order */ |
117 | union smc_host_cursor { /* SMC cursor - an offset in an RMBE */ |
118 | struct { |
119 | u16 reserved; |
120 | u16 wrap; /* window wrap sequence number */ |
121 | u32 count; /* cursor (= offset) part */ |
122 | }; |
123 | #ifdef KERNEL_HAS_ATOMIC64 |
124 | atomic64_t acurs; /* for atomic processing */ |
125 | #else |
126 | u64 acurs; /* for atomic processing */ |
127 | #endif |
128 | } __aligned(8); |
129 | |
130 | /* in host byte order, except for flag bitfields in network byte order */ |
131 | struct smc_host_cdc_msg { /* Connection Data Control message */ |
132 | struct smc_wr_rx_hdr common; /* .type = 0xFE */ |
133 | u8 len; /* length = 44 */ |
134 | u16 seqno; /* connection seq # */ |
135 | u32 token; /* alert_token */ |
136 | union smc_host_cursor prod; /* producer cursor */ |
137 | union smc_host_cursor cons; /* consumer cursor, |
138 | * piggy backed "ack" |
139 | */ |
140 | struct smc_cdc_producer_flags prod_flags; /* conn. tx/rx status */ |
141 | struct smc_cdc_conn_state_flags conn_state_flags; /* peer conn. status*/ |
142 | u8 reserved[18]; |
143 | } __aligned(8); |
144 | |
145 | enum smc_urg_state { |
146 | SMC_URG_VALID = 1, /* data present */ |
147 | SMC_URG_NOTYET = 2, /* data pending */ |
148 | SMC_URG_READ = 3, /* data was already read */ |
149 | }; |
150 | |
151 | struct smc_mark_woken { |
152 | bool woken; |
153 | void *key; |
154 | wait_queue_entry_t wait_entry; |
155 | }; |
156 | |
157 | struct smc_connection { |
158 | struct rb_node alert_node; |
159 | struct smc_link_group *lgr; /* link group of connection */ |
160 | struct smc_link *lnk; /* assigned SMC-R link */ |
161 | u32 alert_token_local; /* unique conn. id */ |
162 | u8 peer_rmbe_idx; /* from tcp handshake */ |
163 | int peer_rmbe_size; /* size of peer rx buffer */ |
164 | atomic_t peer_rmbe_space;/* remaining free bytes in peer |
165 | * rmbe |
166 | */ |
167 | int rtoken_idx; /* idx to peer RMB rkey/addr */ |
168 | |
169 | struct smc_buf_desc *sndbuf_desc; /* send buffer descriptor */ |
170 | struct smc_buf_desc *rmb_desc; /* RMBE descriptor */ |
171 | int rmbe_size_comp; /* compressed notation */ |
172 | int rmbe_update_limit; |
173 | /* lower limit for consumer |
174 | * cursor update |
175 | */ |
176 | |
177 | struct smc_host_cdc_msg local_tx_ctrl; /* host byte order staging |
178 | * buffer for CDC msg send |
179 | * .prod cf. TCP snd_nxt |
180 | * .cons cf. TCP sends ack |
181 | */ |
182 | union smc_host_cursor local_tx_ctrl_fin; |
183 | /* prod crsr - confirmed by peer |
184 | */ |
185 | union smc_host_cursor tx_curs_prep; /* tx - prepared data |
186 | * snd_max..wmem_alloc |
187 | */ |
188 | union smc_host_cursor tx_curs_sent; /* tx - sent data |
189 | * snd_nxt ? |
190 | */ |
191 | union smc_host_cursor tx_curs_fin; /* tx - confirmed by peer |
192 | * snd-wnd-begin ? |
193 | */ |
194 | atomic_t sndbuf_space; /* remaining space in sndbuf */ |
195 | u16 tx_cdc_seq; /* sequence # for CDC send */ |
196 | u16 tx_cdc_seq_fin; /* sequence # - tx completed */ |
197 | spinlock_t send_lock; /* protect wr_sends */ |
198 | atomic_t cdc_pend_tx_wr; /* number of pending tx CDC wqe |
199 | * - inc when post wqe, |
200 | * - dec on polled tx cqe |
201 | */ |
202 | wait_queue_head_t cdc_pend_tx_wq; /* wakeup on no cdc_pend_tx_wr*/ |
203 | struct delayed_work tx_work; /* retry of smc_cdc_msg_send */ |
204 | u32 tx_off; /* base offset in peer rmb */ |
205 | |
206 | struct smc_host_cdc_msg local_rx_ctrl; /* filled during event_handl. |
207 | * .prod cf. TCP rcv_nxt |
208 | * .cons cf. TCP snd_una |
209 | */ |
210 | union smc_host_cursor rx_curs_confirmed; /* confirmed to peer |
211 | * source of snd_una ? |
212 | */ |
213 | union smc_host_cursor urg_curs; /* points at urgent byte */ |
214 | enum smc_urg_state urg_state; |
215 | bool urg_tx_pend; /* urgent data staged */ |
216 | bool urg_rx_skip_pend; |
217 | /* indicate urgent oob data |
218 | * read, but previous regular |
219 | * data still pending |
220 | */ |
221 | char urg_rx_byte; /* urgent byte */ |
222 | bool tx_in_release_sock; |
223 | /* flush pending tx data in |
224 | * sock release_cb() |
225 | */ |
226 | atomic_t bytes_to_rcv; /* arrived data, |
227 | * not yet received |
228 | */ |
229 | atomic_t splice_pending; /* number of spliced bytes |
230 | * pending processing |
231 | */ |
232 | #ifndef KERNEL_HAS_ATOMIC64 |
233 | spinlock_t acurs_lock; /* protect cursors */ |
234 | #endif |
235 | struct work_struct close_work; /* peer sent some closing */ |
236 | struct work_struct abort_work; /* abort the connection */ |
237 | struct tasklet_struct rx_tsklet; /* Receiver tasklet for SMC-D */ |
238 | u8 rx_off; /* receive offset: |
239 | * 0 for SMC-R, 32 for SMC-D |
240 | */ |
241 | u64 peer_token; /* SMC-D token of peer */ |
242 | u8 killed : 1; /* abnormal termination */ |
243 | u8 freed : 1; /* normal termiation */ |
244 | u8 out_of_sync : 1; /* out of sync with peer */ |
245 | }; |
246 | |
247 | struct smc_sock { /* smc sock container */ |
248 | struct sock sk; |
249 | struct socket *clcsock; /* internal tcp socket */ |
250 | void (*clcsk_state_change)(struct sock *sk); |
251 | /* original stat_change fct. */ |
252 | void (*clcsk_data_ready)(struct sock *sk); |
253 | /* original data_ready fct. */ |
254 | void (*clcsk_write_space)(struct sock *sk); |
255 | /* original write_space fct. */ |
256 | void (*clcsk_error_report)(struct sock *sk); |
257 | /* original error_report fct. */ |
258 | struct smc_connection conn; /* smc connection */ |
259 | struct smc_sock *listen_smc; /* listen parent */ |
260 | struct work_struct connect_work; /* handle non-blocking connect*/ |
261 | struct work_struct tcp_listen_work;/* handle tcp socket accepts */ |
262 | struct work_struct smc_listen_work;/* prepare new accept socket */ |
263 | struct list_head accept_q; /* sockets to be accepted */ |
264 | spinlock_t accept_q_lock; /* protects accept_q */ |
265 | bool limit_smc_hs; /* put constraint on handshake */ |
266 | bool use_fallback; /* fallback to tcp */ |
267 | int fallback_rsn; /* reason for fallback */ |
268 | u32 peer_diagnosis; /* decline reason from peer */ |
269 | atomic_t queued_smc_hs; /* queued smc handshakes */ |
270 | struct inet_connection_sock_af_ops af_ops; |
271 | const struct inet_connection_sock_af_ops *ori_af_ops; |
272 | /* original af ops */ |
273 | int sockopt_defer_accept; |
274 | /* sockopt TCP_DEFER_ACCEPT |
275 | * value |
276 | */ |
277 | u8 wait_close_tx_prepared : 1; |
278 | /* shutdown wr or close |
279 | * started, waiting for unsent |
280 | * data to be sent |
281 | */ |
282 | u8 connect_nonblock : 1; |
283 | /* non-blocking connect in |
284 | * flight |
285 | */ |
286 | struct mutex clcsock_release_lock; |
287 | /* protects clcsock of a listen |
288 | * socket |
289 | * */ |
290 | }; |
291 | |
292 | #define smc_sk(ptr) container_of_const(ptr, struct smc_sock, sk) |
293 | |
294 | static inline void smc_init_saved_callbacks(struct smc_sock *smc) |
295 | { |
296 | smc->clcsk_state_change = NULL; |
297 | smc->clcsk_data_ready = NULL; |
298 | smc->clcsk_write_space = NULL; |
299 | smc->clcsk_error_report = NULL; |
300 | } |
301 | |
302 | static inline struct smc_sock *smc_clcsock_user_data(const struct sock *clcsk) |
303 | { |
304 | return (struct smc_sock *) |
305 | ((uintptr_t)clcsk->sk_user_data & ~SK_USER_DATA_NOCOPY); |
306 | } |
307 | |
308 | /* save target_cb in saved_cb, and replace target_cb with new_cb */ |
309 | static inline void smc_clcsock_replace_cb(void (**target_cb)(struct sock *), |
310 | void (*new_cb)(struct sock *), |
311 | void (**saved_cb)(struct sock *)) |
312 | { |
313 | /* only save once */ |
314 | if (!*saved_cb) |
315 | *saved_cb = *target_cb; |
316 | *target_cb = new_cb; |
317 | } |
318 | |
319 | /* restore target_cb to saved_cb, and reset saved_cb to NULL */ |
320 | static inline void smc_clcsock_restore_cb(void (**target_cb)(struct sock *), |
321 | void (**saved_cb)(struct sock *)) |
322 | { |
323 | if (!*saved_cb) |
324 | return; |
325 | *target_cb = *saved_cb; |
326 | *saved_cb = NULL; |
327 | } |
328 | |
329 | extern struct workqueue_struct *smc_hs_wq; /* wq for handshake work */ |
330 | extern struct workqueue_struct *smc_close_wq; /* wq for close work */ |
331 | |
332 | #define SMC_SYSTEMID_LEN 8 |
333 | |
334 | extern u8 local_systemid[SMC_SYSTEMID_LEN]; /* unique system identifier */ |
335 | |
336 | #define ntohll(x) be64_to_cpu(x) |
337 | #define htonll(x) cpu_to_be64(x) |
338 | |
339 | /* convert an u32 value into network byte order, store it into a 3 byte field */ |
340 | static inline void hton24(u8 *net, u32 host) |
341 | { |
342 | __be32 t; |
343 | |
344 | t = cpu_to_be32(host); |
345 | memcpy(net, ((u8 *)&t) + 1, 3); |
346 | } |
347 | |
348 | /* convert a received 3 byte field into host byte order*/ |
349 | static inline u32 ntoh24(u8 *net) |
350 | { |
351 | __be32 t = 0; |
352 | |
353 | memcpy(((u8 *)&t) + 1, net, 3); |
354 | return be32_to_cpu(t); |
355 | } |
356 | |
357 | #ifdef CONFIG_XFRM |
358 | static inline bool using_ipsec(struct smc_sock *smc) |
359 | { |
360 | return (smc->clcsock->sk->sk_policy[0] || |
361 | smc->clcsock->sk->sk_policy[1]) ? true : false; |
362 | } |
363 | #else |
364 | static inline bool using_ipsec(struct smc_sock *smc) |
365 | { |
366 | return false; |
367 | } |
368 | #endif |
369 | |
370 | struct smc_gidlist; |
371 | |
372 | struct sock *smc_accept_dequeue(struct sock *parent, struct socket *new_sock); |
373 | void smc_close_non_accepted(struct sock *sk); |
374 | void smc_fill_gid_list(struct smc_link_group *lgr, |
375 | struct smc_gidlist *gidlist, |
376 | struct smc_ib_device *known_dev, u8 *known_gid); |
377 | |
378 | /* smc handshake limitation interface for netlink */ |
379 | int smc_nl_dump_hs_limitation(struct sk_buff *skb, struct netlink_callback *cb); |
380 | int smc_nl_enable_hs_limitation(struct sk_buff *skb, struct genl_info *info); |
381 | int smc_nl_disable_hs_limitation(struct sk_buff *skb, struct genl_info *info); |
382 | |
383 | static inline void smc_sock_set_flag(struct sock *sk, enum sock_flags flag) |
384 | { |
385 | set_bit(nr: flag, addr: &sk->sk_flags); |
386 | } |
387 | |
388 | #endif /* __SMC_H */ |
389 | |