1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * Copyright (c) 2018 Chelsio Communications, Inc. |
4 | */ |
5 | |
6 | #ifndef __CHTLS_H__ |
7 | #define __CHTLS_H__ |
8 | |
9 | #include <crypto/aes.h> |
10 | #include <crypto/hash.h> |
11 | #include <crypto/sha1.h> |
12 | #include <crypto/sha2.h> |
13 | #include <crypto/authenc.h> |
14 | #include <crypto/ctr.h> |
15 | #include <crypto/gf128mul.h> |
16 | #include <crypto/internal/aead.h> |
17 | #include <crypto/null.h> |
18 | #include <crypto/internal/skcipher.h> |
19 | #include <crypto/aead.h> |
20 | #include <crypto/scatterwalk.h> |
21 | #include <crypto/internal/hash.h> |
22 | #include <linux/tls.h> |
23 | #include <net/tls.h> |
24 | #include <net/tls_prot.h> |
25 | #include <net/tls_toe.h> |
26 | |
27 | #include "t4fw_api.h" |
28 | #include "t4_msg.h" |
29 | #include "cxgb4.h" |
30 | #include "cxgb4_uld.h" |
31 | #include "l2t.h" |
32 | #include "chcr_algo.h" |
33 | #include "chcr_core.h" |
34 | #include "chcr_crypto.h" |
35 | |
36 | #define CHTLS_DRV_VERSION "1.0.0.0-ko" |
37 | |
38 | #define TLS_KEYCTX_RXFLIT_CNT_S 24 |
39 | #define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S) |
40 | |
41 | #define TLS_KEYCTX_RXPROT_VER_S 20 |
42 | #define TLS_KEYCTX_RXPROT_VER_M 0xf |
43 | #define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S) |
44 | |
45 | #define TLS_KEYCTX_RXCIPH_MODE_S 16 |
46 | #define TLS_KEYCTX_RXCIPH_MODE_M 0xf |
47 | #define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S) |
48 | |
49 | #define TLS_KEYCTX_RXAUTH_MODE_S 12 |
50 | #define TLS_KEYCTX_RXAUTH_MODE_M 0xf |
51 | #define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S) |
52 | |
53 | #define TLS_KEYCTX_RXCIAU_CTRL_S 11 |
54 | #define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S) |
55 | |
56 | #define TLS_KEYCTX_RX_SEQCTR_S 9 |
57 | #define TLS_KEYCTX_RX_SEQCTR_M 0x3 |
58 | #define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S) |
59 | |
60 | #define TLS_KEYCTX_RX_VALID_S 8 |
61 | #define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S) |
62 | |
63 | #define TLS_KEYCTX_RXCK_SIZE_S 3 |
64 | #define TLS_KEYCTX_RXCK_SIZE_M 0x7 |
65 | #define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S) |
66 | |
67 | #define TLS_KEYCTX_RXMK_SIZE_S 0 |
68 | #define TLS_KEYCTX_RXMK_SIZE_M 0x7 |
69 | #define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S) |
70 | |
71 | #define KEYCTX_TX_WR_IV_S 55 |
72 | #define KEYCTX_TX_WR_IV_M 0x1ffULL |
73 | #define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S) |
74 | #define KEYCTX_TX_WR_IV_G(x) \ |
75 | (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M) |
76 | |
77 | #define KEYCTX_TX_WR_AAD_S 47 |
78 | #define KEYCTX_TX_WR_AAD_M 0xffULL |
79 | #define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S) |
80 | #define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \ |
81 | KEYCTX_TX_WR_AAD_M) |
82 | |
83 | #define KEYCTX_TX_WR_AADST_S 39 |
84 | #define KEYCTX_TX_WR_AADST_M 0xffULL |
85 | #define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S) |
86 | #define KEYCTX_TX_WR_AADST_G(x) \ |
87 | (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M) |
88 | |
89 | #define KEYCTX_TX_WR_CIPHER_S 30 |
90 | #define KEYCTX_TX_WR_CIPHER_M 0x1ffULL |
91 | #define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S) |
92 | #define KEYCTX_TX_WR_CIPHER_G(x) \ |
93 | (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M) |
94 | |
95 | #define KEYCTX_TX_WR_CIPHERST_S 23 |
96 | #define KEYCTX_TX_WR_CIPHERST_M 0x7f |
97 | #define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S) |
98 | #define KEYCTX_TX_WR_CIPHERST_G(x) \ |
99 | (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M) |
100 | |
101 | #define KEYCTX_TX_WR_AUTH_S 14 |
102 | #define KEYCTX_TX_WR_AUTH_M 0x1ff |
103 | #define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S) |
104 | #define KEYCTX_TX_WR_AUTH_G(x) \ |
105 | (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M) |
106 | |
107 | #define KEYCTX_TX_WR_AUTHST_S 7 |
108 | #define KEYCTX_TX_WR_AUTHST_M 0x7f |
109 | #define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S) |
110 | #define KEYCTX_TX_WR_AUTHST_G(x) \ |
111 | (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M) |
112 | |
113 | #define KEYCTX_TX_WR_AUTHIN_S 0 |
114 | #define KEYCTX_TX_WR_AUTHIN_M 0x7f |
115 | #define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S) |
116 | #define KEYCTX_TX_WR_AUTHIN_G(x) \ |
117 | (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTHIN_M) |
118 | |
119 | struct sge_opaque_hdr { |
120 | void *dev; |
121 | dma_addr_t addr[MAX_SKB_FRAGS + 1]; |
122 | }; |
123 | |
124 | #define MAX_IVS_PAGE 256 |
125 | #define TLS_KEY_CONTEXT_SZ 64 |
126 | #define CIPHER_BLOCK_SIZE 16 |
127 | #define GCM_TAG_SIZE 16 |
128 | #define KEY_ON_MEM_SZ 16 |
129 | #define AEAD_EXPLICIT_DATA_SIZE 8 |
130 | #define 5 |
131 | #define SCMD_CIPH_MODE_AES_GCM 2 |
132 | /* Any MFS size should work and come from openssl */ |
133 | #define TLS_MFS 16384 |
134 | |
135 | #define sizeof(struct rss_header) |
136 | #define TLS_WR_CPL_LEN \ |
137 | (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo)) |
138 | |
139 | enum { |
140 | CHTLS_KEY_CONTEXT_DSGL, |
141 | CHTLS_KEY_CONTEXT_IMM, |
142 | CHTLS_KEY_CONTEXT_DDR, |
143 | }; |
144 | |
145 | enum { |
146 | CHTLS_LISTEN_START, |
147 | CHTLS_LISTEN_STOP, |
148 | }; |
149 | |
150 | /* Flags for return value of CPL message handlers */ |
151 | enum { |
152 | CPL_RET_BUF_DONE = 1, /* buffer processing done */ |
153 | CPL_RET_BAD_MSG = 2, /* bad CPL message */ |
154 | CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */ |
155 | }; |
156 | |
157 | #define LISTEN_INFO_HASH_SIZE 32 |
158 | #define RSPQ_HASH_BITS 5 |
159 | struct listen_info { |
160 | struct listen_info *next; /* Link to next entry */ |
161 | struct sock *sk; /* The listening socket */ |
162 | unsigned int stid; /* The server TID */ |
163 | }; |
164 | |
165 | enum { |
166 | T4_LISTEN_START_PENDING, |
167 | T4_LISTEN_STARTED |
168 | }; |
169 | |
170 | enum csk_flags { |
171 | CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */ |
172 | CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */ |
173 | CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */ |
174 | CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */ |
175 | CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */ |
176 | CSK_ABORT_RPL_PENDING, /* expecting an abort reply */ |
177 | CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */ |
178 | CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */ |
179 | CSK_TX_FAILOVER, /* Tx traffic failing over */ |
180 | CSK_UPDATE_RCV_WND, /* Need to update rcv window */ |
181 | CSK_RST_ABORTED, /* outgoing RST was aborted */ |
182 | CSK_TLS_HANDSHK, /* TLS Handshake */ |
183 | CSK_CONN_INLINE, /* Connection on HW */ |
184 | }; |
185 | |
186 | enum chtls_cdev_state { |
187 | CHTLS_CDEV_STATE_UP = 1 |
188 | }; |
189 | |
190 | struct listen_ctx { |
191 | struct sock *lsk; |
192 | struct chtls_dev *cdev; |
193 | struct sk_buff_head synq; |
194 | u32 state; |
195 | }; |
196 | |
197 | struct key_map { |
198 | unsigned long *addr; |
199 | unsigned int start; |
200 | unsigned int available; |
201 | unsigned int size; |
202 | spinlock_t lock; /* lock for key id request from map */ |
203 | } __packed; |
204 | |
205 | struct tls_scmd { |
206 | u32 seqno_numivs; |
207 | u32 ivgen_hdrlen; |
208 | }; |
209 | |
210 | struct chtls_dev { |
211 | struct tls_toe_device tlsdev; |
212 | struct list_head list; |
213 | struct cxgb4_lld_info *lldi; |
214 | struct pci_dev *pdev; |
215 | struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE]; |
216 | spinlock_t listen_lock; /* lock for listen list */ |
217 | struct net_device **ports; |
218 | struct tid_info *tids; |
219 | unsigned int pfvf; |
220 | const unsigned short *mtus; |
221 | |
222 | struct idr hwtid_idr; |
223 | struct idr stid_idr; |
224 | |
225 | spinlock_t idr_lock ____cacheline_aligned_in_smp; |
226 | |
227 | struct net_device *egr_dev[NCHAN * 2]; |
228 | struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS]; |
229 | struct sk_buff *askb; |
230 | |
231 | struct sk_buff_head deferq; |
232 | struct work_struct deferq_task; |
233 | |
234 | struct list_head list_node; |
235 | struct list_head rcu_node; |
236 | struct list_head na_node; |
237 | unsigned int send_page_order; |
238 | int max_host_sndbuf; |
239 | u32 round_robin_cnt; |
240 | struct key_map kmap; |
241 | unsigned int cdev_state; |
242 | }; |
243 | |
244 | struct chtls_listen { |
245 | struct chtls_dev *cdev; |
246 | struct sock *sk; |
247 | }; |
248 | |
249 | struct chtls_hws { |
250 | struct sk_buff_head sk_recv_queue; |
251 | u8 txqid; |
252 | u8 ofld; |
253 | u16 type; |
254 | u16 rstate; |
255 | u16 keyrpl; |
256 | u16 pldlen; |
257 | u16 rcvpld; |
258 | u16 compute; |
259 | u16 expansion; |
260 | u16 keylen; |
261 | u16 pdus; |
262 | u16 adjustlen; |
263 | u16 ivsize; |
264 | u16 txleft; |
265 | u32 mfs; |
266 | s32 txkey; |
267 | s32 rxkey; |
268 | u32 fcplenmax; |
269 | u32 copied_seq; |
270 | u64 tx_seq_no; |
271 | struct tls_scmd scmd; |
272 | union { |
273 | struct tls12_crypto_info_aes_gcm_128 aes_gcm_128; |
274 | struct tls12_crypto_info_aes_gcm_256 aes_gcm_256; |
275 | } crypto_info; |
276 | }; |
277 | |
278 | struct chtls_sock { |
279 | struct sock *sk; |
280 | struct chtls_dev *cdev; |
281 | struct l2t_entry *l2t_entry; /* pointer to the L2T entry */ |
282 | struct net_device *egress_dev; /* TX_CHAN for act open retry */ |
283 | |
284 | struct sk_buff_head txq; |
285 | struct sk_buff *wr_skb_head; |
286 | struct sk_buff *wr_skb_tail; |
287 | struct sk_buff *ctrl_skb_cache; |
288 | struct sk_buff *txdata_skb_cache; /* abort path messages */ |
289 | struct kref kref; |
290 | unsigned long flags; |
291 | u32 opt2; |
292 | u32 wr_credits; |
293 | u32 wr_unacked; |
294 | u32 wr_max_credits; |
295 | u32 wr_nondata; |
296 | u32 hwtid; /* TCP Control Block ID */ |
297 | u32 txq_idx; |
298 | u32 ; |
299 | u32 tid; |
300 | u32 idr; |
301 | u32 mss; |
302 | u32 ulp_mode; |
303 | u32 tx_chan; |
304 | u32 rx_chan; |
305 | u32 sndbuf; |
306 | u32 txplen_max; |
307 | u32 mtu_idx; /* MTU table index */ |
308 | u32 smac_idx; |
309 | u8 port_id; |
310 | u8 tos; |
311 | u16 resv2; |
312 | u32 delack_mode; |
313 | u32 delack_seq; |
314 | u32 snd_win; |
315 | u32 rcv_win; |
316 | |
317 | void *passive_reap_next; /* placeholder for passive */ |
318 | struct chtls_hws tlshws; |
319 | struct synq { |
320 | struct sk_buff *next; |
321 | struct sk_buff *prev; |
322 | } synq; |
323 | struct listen_ctx *listen_ctx; |
324 | }; |
325 | |
326 | struct tls_hdr { |
327 | u8 type; |
328 | u16 version; |
329 | u16 length; |
330 | } __packed; |
331 | |
332 | struct tlsrx_cmp_hdr { |
333 | u8 type; |
334 | u16 version; |
335 | u16 length; |
336 | |
337 | u64 tls_seq; |
338 | u16 reserved1; |
339 | u8 res_to_mac_error; |
340 | } __packed; |
341 | |
342 | /* res_to_mac_error fields */ |
343 | #define TLSRX_HDR_PKT_INT_ERROR_S 4 |
344 | #define TLSRX_HDR_PKT_INT_ERROR_M 0x1 |
345 | #define TLSRX_HDR_PKT_INT_ERROR_V(x) \ |
346 | ((x) << TLSRX_HDR_PKT_INT_ERROR_S) |
347 | #define TLSRX_HDR_PKT_INT_ERROR_G(x) \ |
348 | (((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M) |
349 | #define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U) |
350 | |
351 | #define TLSRX_HDR_PKT_SPP_ERROR_S 3 |
352 | #define TLSRX_HDR_PKT_SPP_ERROR_M 0x1 |
353 | #define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR) |
354 | #define TLSRX_HDR_PKT_SPP_ERROR_G(x) \ |
355 | (((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M) |
356 | #define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U) |
357 | |
358 | #define TLSRX_HDR_PKT_CCDX_ERROR_S 2 |
359 | #define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1 |
360 | #define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S) |
361 | #define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \ |
362 | (((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M) |
363 | #define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U) |
364 | |
365 | #define TLSRX_HDR_PKT_PAD_ERROR_S 1 |
366 | #define TLSRX_HDR_PKT_PAD_ERROR_M 0x1 |
367 | #define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S) |
368 | #define TLSRX_HDR_PKT_PAD_ERROR_G(x) \ |
369 | (((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M) |
370 | #define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U) |
371 | |
372 | #define TLSRX_HDR_PKT_MAC_ERROR_S 0 |
373 | #define TLSRX_HDR_PKT_MAC_ERROR_M 0x1 |
374 | #define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR) |
375 | #define TLSRX_HDR_PKT_MAC_ERROR_G(x) \ |
376 | (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M) |
377 | #define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U) |
378 | |
379 | #define TLSRX_HDR_PKT_ERROR_M 0x1F |
380 | #define CONTENT_TYPE_ERROR 0x7F |
381 | |
382 | struct ulp_mem_rw { |
383 | __be32 cmd; |
384 | __be32 len16; /* command length */ |
385 | __be32 dlen; /* data length in 32-byte units */ |
386 | __be32 lock_addr; |
387 | }; |
388 | |
389 | struct tls_key_wr { |
390 | __be32 op_to_compl; |
391 | __be32 flowid_len16; |
392 | __be32 ftid; |
393 | u8 reneg_to_write_rx; |
394 | u8 protocol; |
395 | __be16 mfs; |
396 | }; |
397 | |
398 | struct tls_key_req { |
399 | struct tls_key_wr wr; |
400 | struct ulp_mem_rw req; |
401 | struct ulptx_idata sc_imm; |
402 | }; |
403 | |
404 | /* |
405 | * This lives in skb->cb and is used to chain WRs in a linked list. |
406 | */ |
407 | struct wr_skb_cb { |
408 | struct l2t_skb_cb l2t; /* reserve space for l2t CB */ |
409 | struct sk_buff *next_wr; /* next write request */ |
410 | }; |
411 | |
412 | /* Per-skb backlog handler. Run when a socket's backlog is processed. */ |
413 | struct blog_skb_cb { |
414 | void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb); |
415 | struct chtls_dev *cdev; |
416 | }; |
417 | |
418 | /* |
419 | * Similar to tcp_skb_cb but with ULP elements added to support TLS, |
420 | * etc. |
421 | */ |
422 | struct ulp_skb_cb { |
423 | struct wr_skb_cb wr; /* reserve space for write request */ |
424 | u16 flags; /* TCP-like flags */ |
425 | u8 psh; |
426 | u8 ulp_mode; /* ULP mode/submode of sk_buff */ |
427 | u32 seq; /* TCP sequence number */ |
428 | union { /* ULP-specific fields */ |
429 | struct { |
430 | u8 type; |
431 | u8 ofld; |
432 | u8 iv; |
433 | } tls; |
434 | } ulp; |
435 | }; |
436 | |
437 | #define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0])) |
438 | #define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb) |
439 | |
440 | /* |
441 | * Flags for ulp_skb_cb.flags. |
442 | */ |
443 | enum { |
444 | ULPCB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */ |
445 | ULPCB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */ |
446 | ULPCB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */ |
447 | ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */ |
448 | ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */ |
449 | ULPCB_FLAG_URG = 1 << 5, /* urgent data */ |
450 | ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */ |
451 | ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */ |
452 | }; |
453 | |
454 | /* The ULP mode/submode of an skbuff */ |
455 | #define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode) |
456 | #define TCP_PAGE(sk) (sk->sk_frag.page) |
457 | #define TCP_OFF(sk) (sk->sk_frag.offset) |
458 | |
459 | static inline struct chtls_dev *to_chtls_dev(struct tls_toe_device *tlsdev) |
460 | { |
461 | return container_of(tlsdev, struct chtls_dev, tlsdev); |
462 | } |
463 | |
464 | static inline void csk_set_flag(struct chtls_sock *csk, |
465 | enum csk_flags flag) |
466 | { |
467 | __set_bit(flag, &csk->flags); |
468 | } |
469 | |
470 | static inline void csk_reset_flag(struct chtls_sock *csk, |
471 | enum csk_flags flag) |
472 | { |
473 | __clear_bit(flag, &csk->flags); |
474 | } |
475 | |
476 | static inline bool csk_conn_inline(const struct chtls_sock *csk) |
477 | { |
478 | return test_bit(CSK_CONN_INLINE, &csk->flags); |
479 | } |
480 | |
481 | static inline int csk_flag(const struct sock *sk, enum csk_flags flag) |
482 | { |
483 | struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); |
484 | |
485 | if (!csk_conn_inline(csk)) |
486 | return 0; |
487 | return test_bit(flag, &csk->flags); |
488 | } |
489 | |
490 | static inline int csk_flag_nochk(const struct chtls_sock *csk, |
491 | enum csk_flags flag) |
492 | { |
493 | return test_bit(flag, &csk->flags); |
494 | } |
495 | |
496 | static inline void *cplhdr(struct sk_buff *skb) |
497 | { |
498 | return skb->data; |
499 | } |
500 | |
501 | static inline int is_neg_adv(unsigned int status) |
502 | { |
503 | return status == CPL_ERR_RTX_NEG_ADVICE || |
504 | status == CPL_ERR_KEEPALV_NEG_ADVICE || |
505 | status == CPL_ERR_PERSIST_NEG_ADVICE; |
506 | } |
507 | |
508 | static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *), |
509 | struct sock *sk, |
510 | struct sk_buff *skb) |
511 | { |
512 | skb_reset_mac_header(skb); |
513 | skb_reset_network_header(skb); |
514 | skb_reset_transport_header(skb); |
515 | |
516 | bh_lock_sock(sk); |
517 | if (unlikely(sock_owned_by_user(sk))) { |
518 | BLOG_SKB_CB(skb)->backlog_rcv = fn; |
519 | __sk_add_backlog(sk, skb); |
520 | } else { |
521 | fn(sk, skb); |
522 | } |
523 | bh_unlock_sock(sk); |
524 | } |
525 | |
526 | static inline void chtls_sock_free(struct kref *ref) |
527 | { |
528 | struct chtls_sock *csk = container_of(ref, struct chtls_sock, |
529 | kref); |
530 | kfree(objp: csk); |
531 | } |
532 | |
533 | static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk) |
534 | { |
535 | kref_put(kref: &csk->kref, release: chtls_sock_free); |
536 | } |
537 | |
538 | static inline void __chtls_sock_get(const char *fn, |
539 | struct chtls_sock *csk) |
540 | { |
541 | kref_get(kref: &csk->kref); |
542 | } |
543 | |
544 | static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp, |
545 | struct sk_buff *skb, int through_l2t) |
546 | { |
547 | struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); |
548 | |
549 | if (through_l2t) { |
550 | /* send through L2T */ |
551 | cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); |
552 | } else { |
553 | /* send directly */ |
554 | cxgb4_ofld_send(csk->egress_dev, skb); |
555 | } |
556 | } |
557 | |
558 | typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *); |
559 | extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS]; |
560 | void chtls_install_cpl_ops(struct sock *sk); |
561 | int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi); |
562 | void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk); |
563 | int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk); |
564 | void chtls_close(struct sock *sk, long timeout); |
565 | int chtls_disconnect(struct sock *sk, int flags); |
566 | void chtls_shutdown(struct sock *sk, int how); |
567 | void chtls_destroy_sock(struct sock *sk); |
568 | int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); |
569 | int chtls_recvmsg(struct sock *sk, struct msghdr *msg, |
570 | size_t len, int flags, int *addr_len); |
571 | void chtls_splice_eof(struct socket *sock); |
572 | int send_tx_flowc_wr(struct sock *sk, int compl, |
573 | u32 snd_nxt, u32 rcv_nxt); |
574 | void chtls_tcp_push(struct sock *sk, int flags); |
575 | int chtls_push_frames(struct chtls_sock *csk, int comp); |
576 | int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val); |
577 | void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word, |
578 | u64 mask, u64 val, u8 cookie, |
579 | int through_l2t); |
580 | int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode, int cipher_type); |
581 | void chtls_set_quiesce_ctrl(struct sock *sk, int val); |
582 | void skb_entail(struct sock *sk, struct sk_buff *skb, int flags); |
583 | unsigned int keyid_to_addr(int start_addr, int keyid); |
584 | void free_tls_keyid(struct sock *sk); |
585 | #endif |
586 | |