1/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
2/*
3 * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
4 */
5
6#ifndef _EFA_IO_H_
7#define _EFA_IO_H_
8
9#define EFA_IO_TX_DESC_NUM_BUFS 2
10#define EFA_IO_TX_DESC_NUM_RDMA_BUFS 1
11#define EFA_IO_TX_DESC_INLINE_MAX_SIZE 32
12#define EFA_IO_TX_DESC_IMM_DATA_SIZE 4
13#define EFA_IO_TX_DESC_INLINE_PBL_SIZE 1
14
15enum efa_io_queue_type {
16 /* send queue (of a QP) */
17 EFA_IO_SEND_QUEUE = 1,
18 /* recv queue (of a QP) */
19 EFA_IO_RECV_QUEUE = 2,
20};
21
22enum efa_io_send_op_type {
23 /* send message */
24 EFA_IO_SEND = 0,
25 /* RDMA read */
26 EFA_IO_RDMA_READ = 1,
27 /* RDMA write */
28 EFA_IO_RDMA_WRITE = 2,
29 /* Fast MR registration */
30 EFA_IO_FAST_REG = 3,
31 /* Fast MR invalidation */
32 EFA_IO_FAST_INV = 4,
33};
34
35enum efa_io_comp_status {
36 /* Successful completion */
37 EFA_IO_COMP_STATUS_OK = 0,
38 /* Flushed during QP destroy */
39 EFA_IO_COMP_STATUS_FLUSHED = 1,
40 /* Internal QP error */
41 EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2,
42 /* Unsupported operation */
43 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNSUPPORTED_OP = 3,
44 /* Bad AH */
45 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH = 4,
46 /* LKEY not registered or does not match IOVA */
47 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5,
48 /* Message too long */
49 EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH = 6,
50 /* RKEY not registered or does not match remote IOVA */
51 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7,
52 /* Connection was reset by remote side */
53 EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT = 8,
54 /* Bad dest QP number (QP does not exist or is in error state) */
55 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9,
56 /* Destination resource not ready (no WQEs posted on RQ) */
57 EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR = 10,
58 /* Receiver SGL too short */
59 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH = 11,
60 /* Unexpected status returned by responder */
61 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS = 12,
62 /* Unresponsive remote - was previously responsive */
63 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13,
64 /* No valid AH at remote side (required for RDMA operations) */
65 EFA_IO_COMP_STATUS_REMOTE_ERROR_UNKNOWN_PEER = 14,
66 /* Unreachable remote - never received a response */
67 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE = 15,
68};
69
70enum efa_io_frwr_pbl_mode {
71 EFA_IO_FRWR_INLINE_PBL = 0,
72 EFA_IO_FRWR_DIRECT_PBL = 1,
73};
74
75struct efa_io_tx_meta_desc {
76 /* Verbs-generated Request ID */
77 u16 req_id;
78
79 /*
80 * control flags
81 * 3:0 : op_type - enum efa_io_send_op_type
82 * 4 : has_imm - immediate_data field carries valid
83 * data.
84 * 5 : inline_msg - inline mode - inline message data
85 * follows this descriptor (no buffer descriptors).
86 * Note that it is different from immediate data
87 * 6 : meta_extension - Extended metadata. MBZ
88 * 7 : meta_desc - Indicates metadata descriptor.
89 * Must be set.
90 */
91 u8 ctrl1;
92
93 /*
94 * control flags
95 * 0 : phase
96 * 1 : reserved25 - MBZ
97 * 2 : first - Indicates first descriptor in
98 * transaction. Must be set.
99 * 3 : last - Indicates last descriptor in
100 * transaction. Must be set.
101 * 4 : comp_req - Indicates whether completion should
102 * be posted, after packet is transmitted. Valid only
103 * for the first descriptor
104 * 7:5 : reserved29 - MBZ
105 */
106 u8 ctrl2;
107
108 u16 dest_qp_num;
109
110 /*
111 * If inline_msg bit is set, length of inline message in bytes,
112 * otherwise length of SGL (number of buffers).
113 */
114 u16 length;
115
116 /*
117 * immediate data: if has_imm is set, then this field is included within
118 * Tx message and reported in remote Rx completion.
119 */
120 u32 immediate_data;
121
122 u16 ah;
123
124 u16 reserved;
125
126 /* Queue key */
127 u32 qkey;
128
129 u8 reserved2[12];
130};
131
132/*
133 * Tx queue buffer descriptor, for any transport type. Preceded by metadata
134 * descriptor.
135 */
136struct efa_io_tx_buf_desc {
137 /* length in bytes */
138 u32 length;
139
140 /*
141 * 23:0 : lkey - local memory translation key
142 * 31:24 : reserved - MBZ
143 */
144 u32 lkey;
145
146 /* Buffer address bits[31:0] */
147 u32 buf_addr_lo;
148
149 /* Buffer address bits[63:32] */
150 u32 buf_addr_hi;
151};
152
153struct efa_io_remote_mem_addr {
154 /* length in bytes */
155 u32 length;
156
157 /* remote memory translation key */
158 u32 rkey;
159
160 /* Buffer address bits[31:0] */
161 u32 buf_addr_lo;
162
163 /* Buffer address bits[63:32] */
164 u32 buf_addr_hi;
165};
166
167struct efa_io_rdma_req {
168 /* Remote memory address */
169 struct efa_io_remote_mem_addr remote_mem;
170
171 /* Local memory address */
172 struct efa_io_tx_buf_desc local_mem[1];
173};
174
175struct efa_io_fast_mr_reg_req {
176 /* Updated local key of the MR after lkey/rkey increment */
177 u32 lkey;
178
179 /*
180 * permissions
181 * 0 : local_write_enable - Local write permissions:
182 * must be set for RQ buffers and buffers posted for
183 * RDMA Read requests
184 * 1 : remote_write_enable - Remote write
185 * permissions: must be set to enable RDMA write to
186 * the region
187 * 2 : remote_read_enable - Remote read permissions:
188 * must be set to enable RDMA read from the region
189 * 7:3 : reserved2 - MBZ
190 */
191 u8 permissions;
192
193 /*
194 * control flags
195 * 4:0 : phys_page_size_shift - page size is (1 <<
196 * phys_page_size_shift)
197 * 6:5 : pbl_mode - enum efa_io_frwr_pbl_mode
198 * 7 : reserved - MBZ
199 */
200 u8 flags;
201
202 /* MBZ */
203 u8 reserved[2];
204
205 /* IO Virtual Address associated with this MR */
206 u64 iova;
207
208 /* Memory region length, in bytes */
209 u64 mr_length;
210
211 /* Physical Buffer List, each element is page-aligned. */
212 union {
213 /*
214 * Inline array of physical page addresses (optimization
215 * for short region activation).
216 */
217 u64 inline_array[1];
218
219 /* points to PBL (Currently only direct) */
220 u64 dma_addr;
221 } pbl;
222};
223
224struct efa_io_fast_mr_inv_req {
225 /* Local key of the MR to invalidate */
226 u32 lkey;
227
228 /* MBZ */
229 u8 reserved[28];
230};
231
232/*
233 * Tx WQE, composed of tx meta descriptors followed by either tx buffer
234 * descriptors or inline data
235 */
236struct efa_io_tx_wqe {
237 /* TX meta */
238 struct efa_io_tx_meta_desc meta;
239
240 union {
241 /* Send buffer descriptors */
242 struct efa_io_tx_buf_desc sgl[2];
243
244 u8 inline_data[32];
245
246 /* RDMA local and remote memory addresses */
247 struct efa_io_rdma_req rdma_req;
248
249 /* Fast registration */
250 struct efa_io_fast_mr_reg_req reg_mr_req;
251
252 /* Fast invalidation */
253 struct efa_io_fast_mr_inv_req inv_mr_req;
254 } data;
255};
256
257/*
258 * Rx buffer descriptor; RX WQE is composed of one or more RX buffer
259 * descriptors.
260 */
261struct efa_io_rx_desc {
262 /* Buffer address bits[31:0] */
263 u32 buf_addr_lo;
264
265 /* Buffer Pointer[63:32] */
266 u32 buf_addr_hi;
267
268 /* Verbs-generated request id. */
269 u16 req_id;
270
271 /* Length in bytes. */
272 u16 length;
273
274 /*
275 * LKey and control flags
276 * 23:0 : lkey
277 * 29:24 : reserved - MBZ
278 * 30 : first - Indicates first descriptor in WQE
279 * 31 : last - Indicates last descriptor in WQE
280 */
281 u32 lkey_ctrl;
282};
283
284/* Common IO completion descriptor */
285struct efa_io_cdesc_common {
286 /*
287 * verbs-generated request ID, as provided in the completed tx or rx
288 * descriptor.
289 */
290 u16 req_id;
291
292 u8 status;
293
294 /*
295 * flags
296 * 0 : phase - Phase bit
297 * 2:1 : q_type - enum efa_io_queue_type: send/recv
298 * 3 : has_imm - indicates that immediate data is
299 * present - for RX completions only
300 * 6:4 : op_type - enum efa_io_send_op_type
301 * 7 : unsolicited - indicates that there is no
302 * matching request - for RDMA with imm. RX only
303 */
304 u8 flags;
305
306 /* local QP number */
307 u16 qp_num;
308};
309
310/* Tx completion descriptor */
311struct efa_io_tx_cdesc {
312 /* Common completion info */
313 struct efa_io_cdesc_common common;
314
315 /* MBZ */
316 u16 reserved16;
317};
318
319/* Rx Completion Descriptor */
320struct efa_io_rx_cdesc {
321 /* Common completion info */
322 struct efa_io_cdesc_common common;
323
324 /* Transferred length bits[15:0] */
325 u16 length;
326
327 /* Remote Address Handle FW index, 0xFFFF indicates invalid ah */
328 u16 ah;
329
330 u16 src_qp_num;
331
332 /* Immediate data */
333 u32 imm;
334};
335
336/* Rx Completion Descriptor RDMA write info */
337struct efa_io_rx_cdesc_rdma_write {
338 /* Transferred length bits[31:16] */
339 u16 length_hi;
340};
341
342/* Extended Rx Completion Descriptor */
343struct efa_io_rx_cdesc_ex {
344 /* Base RX completion info */
345 struct efa_io_rx_cdesc base;
346
347 union {
348 struct efa_io_rx_cdesc_rdma_write rdma_write;
349
350 /*
351 * Valid only in case of unknown AH (0xFFFF) and CQ
352 * set_src_addr is enabled.
353 */
354 u8 src_addr[16];
355 } u;
356};
357
358/* tx_meta_desc */
359#define EFA_IO_TX_META_DESC_OP_TYPE_MASK GENMASK(3, 0)
360#define EFA_IO_TX_META_DESC_HAS_IMM_MASK BIT(4)
361#define EFA_IO_TX_META_DESC_INLINE_MSG_MASK BIT(5)
362#define EFA_IO_TX_META_DESC_META_EXTENSION_MASK BIT(6)
363#define EFA_IO_TX_META_DESC_META_DESC_MASK BIT(7)
364#define EFA_IO_TX_META_DESC_PHASE_MASK BIT(0)
365#define EFA_IO_TX_META_DESC_FIRST_MASK BIT(2)
366#define EFA_IO_TX_META_DESC_LAST_MASK BIT(3)
367#define EFA_IO_TX_META_DESC_COMP_REQ_MASK BIT(4)
368
369/* tx_buf_desc */
370#define EFA_IO_TX_BUF_DESC_LKEY_MASK GENMASK(23, 0)
371
372/* fast_mr_reg_req */
373#define EFA_IO_FAST_MR_REG_REQ_LOCAL_WRITE_ENABLE_MASK BIT(0)
374#define EFA_IO_FAST_MR_REG_REQ_REMOTE_WRITE_ENABLE_MASK BIT(1)
375#define EFA_IO_FAST_MR_REG_REQ_REMOTE_READ_ENABLE_MASK BIT(2)
376#define EFA_IO_FAST_MR_REG_REQ_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(4, 0)
377#define EFA_IO_FAST_MR_REG_REQ_PBL_MODE_MASK GENMASK(6, 5)
378
379/* rx_desc */
380#define EFA_IO_RX_DESC_LKEY_MASK GENMASK(23, 0)
381#define EFA_IO_RX_DESC_FIRST_MASK BIT(30)
382#define EFA_IO_RX_DESC_LAST_MASK BIT(31)
383
384/* cdesc_common */
385#define EFA_IO_CDESC_COMMON_PHASE_MASK BIT(0)
386#define EFA_IO_CDESC_COMMON_Q_TYPE_MASK GENMASK(2, 1)
387#define EFA_IO_CDESC_COMMON_HAS_IMM_MASK BIT(3)
388#define EFA_IO_CDESC_COMMON_OP_TYPE_MASK GENMASK(6, 4)
389#define EFA_IO_CDESC_COMMON_UNSOLICITED_MASK BIT(7)
390
391#endif /* _EFA_IO_H_ */
392

source code of linux/drivers/infiniband/hw/efa/efa_io_defs.h