1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2015 - 2021 Intel Corporation */ |
3 | #include "osdep.h" |
4 | #include "defs.h" |
5 | #include "user.h" |
6 | #include "irdma.h" |
7 | |
8 | /** |
9 | * irdma_set_fragment - set fragment in wqe |
10 | * @wqe: wqe for setting fragment |
11 | * @offset: offset value |
12 | * @sge: sge length and stag |
13 | * @valid: The wqe valid |
14 | */ |
15 | static void irdma_set_fragment(__le64 *wqe, u32 offset, struct ib_sge *sge, |
16 | u8 valid) |
17 | { |
18 | if (sge) { |
19 | set_64bit_val(wqe_words: wqe, byte_index: offset, |
20 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr)); |
21 | set_64bit_val(wqe_words: wqe, byte_index: offset + 8, |
22 | FIELD_PREP(IRDMAQPSQ_VALID, valid) | |
23 | FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) | |
24 | FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey)); |
25 | } else { |
26 | set_64bit_val(wqe_words: wqe, byte_index: offset, val: 0); |
27 | set_64bit_val(wqe_words: wqe, byte_index: offset + 8, |
28 | FIELD_PREP(IRDMAQPSQ_VALID, valid)); |
29 | } |
30 | } |
31 | |
32 | /** |
33 | * irdma_set_fragment_gen_1 - set fragment in wqe |
34 | * @wqe: wqe for setting fragment |
35 | * @offset: offset value |
36 | * @sge: sge length and stag |
37 | * @valid: wqe valid flag |
38 | */ |
39 | static void irdma_set_fragment_gen_1(__le64 *wqe, u32 offset, |
40 | struct ib_sge *sge, u8 valid) |
41 | { |
42 | if (sge) { |
43 | set_64bit_val(wqe_words: wqe, byte_index: offset, |
44 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr)); |
45 | set_64bit_val(wqe_words: wqe, byte_index: offset + 8, |
46 | FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) | |
47 | FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey)); |
48 | } else { |
49 | set_64bit_val(wqe_words: wqe, byte_index: offset, val: 0); |
50 | set_64bit_val(wqe_words: wqe, byte_index: offset + 8, val: 0); |
51 | } |
52 | } |
53 | |
54 | /** |
55 | * irdma_nop_1 - insert a NOP wqe |
56 | * @qp: hw qp ptr |
57 | */ |
58 | static int irdma_nop_1(struct irdma_qp_uk *qp) |
59 | { |
60 | u64 hdr; |
61 | __le64 *wqe; |
62 | u32 wqe_idx; |
63 | bool signaled = false; |
64 | |
65 | if (!qp->sq_ring.head) |
66 | return -EINVAL; |
67 | |
68 | wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); |
69 | wqe = qp->sq_base[wqe_idx].elem; |
70 | |
71 | qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA; |
72 | |
73 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: 0); |
74 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: 0); |
75 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: 0); |
76 | |
77 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | |
78 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) | |
79 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); |
80 | |
81 | /* make sure WQE is written before valid bit is set */ |
82 | dma_wmb(); |
83 | |
84 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
85 | |
86 | return 0; |
87 | } |
88 | |
89 | /** |
90 | * irdma_clr_wqes - clear next 128 sq entries |
91 | * @qp: hw qp ptr |
92 | * @qp_wqe_idx: wqe_idx |
93 | */ |
94 | void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx) |
95 | { |
96 | struct irdma_qp_quanta *sq; |
97 | u32 wqe_idx; |
98 | |
99 | if (!(qp_wqe_idx & 0x7F)) { |
100 | wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size; |
101 | sq = qp->sq_base + wqe_idx; |
102 | if (wqe_idx) |
103 | memset(sq, qp->swqe_polarity ? 0 : 0xFF, |
104 | 128 * sizeof(*sq)); |
105 | else |
106 | memset(sq, qp->swqe_polarity ? 0xFF : 0, |
107 | 128 * sizeof(*sq)); |
108 | } |
109 | } |
110 | |
111 | /** |
112 | * irdma_uk_qp_post_wr - ring doorbell |
113 | * @qp: hw qp ptr |
114 | */ |
115 | void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp) |
116 | { |
117 | u64 temp; |
118 | u32 hw_sq_tail; |
119 | u32 sw_sq_head; |
120 | |
121 | /* valid bit is written and loads completed before reading shadow */ |
122 | mb(); |
123 | |
124 | /* read the doorbell shadow area */ |
125 | get_64bit_val(wqe_words: qp->shadow_area, byte_index: 0, val: &temp); |
126 | |
127 | hw_sq_tail = (u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp); |
128 | sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); |
129 | if (sw_sq_head != qp->initial_ring.head) { |
130 | if (sw_sq_head != hw_sq_tail) { |
131 | if (sw_sq_head > qp->initial_ring.head) { |
132 | if (hw_sq_tail >= qp->initial_ring.head && |
133 | hw_sq_tail < sw_sq_head) |
134 | writel(val: qp->qp_id, addr: qp->wqe_alloc_db); |
135 | } else { |
136 | if (hw_sq_tail >= qp->initial_ring.head || |
137 | hw_sq_tail < sw_sq_head) |
138 | writel(val: qp->qp_id, addr: qp->wqe_alloc_db); |
139 | } |
140 | } |
141 | } |
142 | |
143 | qp->initial_ring.head = qp->sq_ring.head; |
144 | } |
145 | |
146 | /** |
147 | * irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go |
148 | * @qp: hw qp ptr |
149 | * @wqe_idx: return wqe index |
150 | * @quanta: size of WR in quanta |
151 | * @total_size: size of WR in bytes |
152 | * @info: info on WR |
153 | */ |
154 | __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, |
155 | u16 quanta, u32 total_size, |
156 | struct irdma_post_sq_info *info) |
157 | { |
158 | __le64 *wqe; |
159 | __le64 *wqe_0 = NULL; |
160 | u16 avail_quanta; |
161 | u16 i; |
162 | |
163 | avail_quanta = qp->uk_attrs->max_hw_sq_chunk - |
164 | (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) % |
165 | qp->uk_attrs->max_hw_sq_chunk); |
166 | if (quanta <= avail_quanta) { |
167 | /* WR fits in current chunk */ |
168 | if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) |
169 | return NULL; |
170 | } else { |
171 | /* Need to pad with NOP */ |
172 | if (quanta + avail_quanta > |
173 | IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) |
174 | return NULL; |
175 | |
176 | for (i = 0; i < avail_quanta; i++) { |
177 | irdma_nop_1(qp); |
178 | IRDMA_RING_MOVE_HEAD_NOCHECK(qp->sq_ring); |
179 | } |
180 | } |
181 | |
182 | *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); |
183 | if (!*wqe_idx) |
184 | qp->swqe_polarity = !qp->swqe_polarity; |
185 | |
186 | IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta); |
187 | |
188 | wqe = qp->sq_base[*wqe_idx].elem; |
189 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 && |
190 | (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { |
191 | wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; |
192 | wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity ? 0 : 1)); |
193 | } |
194 | qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; |
195 | qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; |
196 | qp->sq_wrtrk_array[*wqe_idx].quanta = quanta; |
197 | |
198 | return wqe; |
199 | } |
200 | |
201 | /** |
202 | * irdma_qp_get_next_recv_wqe - get next qp's rcv wqe |
203 | * @qp: hw qp ptr |
204 | * @wqe_idx: return wqe index |
205 | */ |
206 | __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx) |
207 | { |
208 | __le64 *wqe; |
209 | int ret_code; |
210 | |
211 | if (IRDMA_RING_FULL_ERR(qp->rq_ring)) |
212 | return NULL; |
213 | |
214 | IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code); |
215 | if (ret_code) |
216 | return NULL; |
217 | |
218 | if (!*wqe_idx) |
219 | qp->rwqe_polarity = !qp->rwqe_polarity; |
220 | /* rq_wqe_size_multiplier is no of 32 byte quanta in one rq wqe */ |
221 | wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem; |
222 | |
223 | return wqe; |
224 | } |
225 | |
226 | /** |
227 | * irdma_uk_rdma_write - rdma write operation |
228 | * @qp: hw qp ptr |
229 | * @info: post sq information |
230 | * @post_sq: flag to post sq |
231 | */ |
232 | int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, |
233 | bool post_sq) |
234 | { |
235 | u64 hdr; |
236 | __le64 *wqe; |
237 | struct irdma_rdma_write *op_info; |
238 | u32 i, wqe_idx; |
239 | u32 total_size = 0, byte_off; |
240 | int ret_code; |
241 | u32 frag_cnt, addl_frag_cnt; |
242 | bool read_fence = false; |
243 | u16 quanta; |
244 | |
245 | op_info = &info->op.rdma_write; |
246 | if (op_info->num_lo_sges > qp->max_sq_frag_cnt) |
247 | return -EINVAL; |
248 | |
249 | for (i = 0; i < op_info->num_lo_sges; i++) |
250 | total_size += op_info->lo_sg_list[i].length; |
251 | |
252 | read_fence |= info->read_fence; |
253 | |
254 | if (info->imm_data_valid) |
255 | frag_cnt = op_info->num_lo_sges + 1; |
256 | else |
257 | frag_cnt = op_info->num_lo_sges; |
258 | addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; |
259 | ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, quanta: &quanta); |
260 | if (ret_code) |
261 | return ret_code; |
262 | |
263 | wqe = irdma_qp_get_next_send_wqe(qp, wqe_idx: &wqe_idx, quanta, total_size, |
264 | info); |
265 | if (!wqe) |
266 | return -ENOMEM; |
267 | |
268 | irdma_clr_wqes(qp, qp_wqe_idx: wqe_idx); |
269 | |
270 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
271 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); |
272 | |
273 | if (info->imm_data_valid) { |
274 | set_64bit_val(wqe_words: wqe, byte_index: 0, |
275 | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); |
276 | i = 0; |
277 | } else { |
278 | qp->wqe_ops.iw_set_fragment(wqe, 0, |
279 | op_info->lo_sg_list, |
280 | qp->swqe_polarity); |
281 | i = 1; |
282 | } |
283 | |
284 | for (byte_off = 32; i < op_info->num_lo_sges; i++) { |
285 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, |
286 | &op_info->lo_sg_list[i], |
287 | qp->swqe_polarity); |
288 | byte_off += 16; |
289 | } |
290 | |
291 | /* if not an odd number set valid bit in next fragment */ |
292 | if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) && |
293 | frag_cnt) { |
294 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, |
295 | qp->swqe_polarity); |
296 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) |
297 | ++addl_frag_cnt; |
298 | } |
299 | |
300 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | |
301 | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | |
302 | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) | |
303 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) | |
304 | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | |
305 | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | |
306 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | |
307 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | |
308 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); |
309 | |
310 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ |
311 | |
312 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
313 | |
314 | if (post_sq) |
315 | irdma_uk_qp_post_wr(qp); |
316 | |
317 | return 0; |
318 | } |
319 | |
320 | /** |
321 | * irdma_uk_rdma_read - rdma read command |
322 | * @qp: hw qp ptr |
323 | * @info: post sq information |
324 | * @inv_stag: flag for inv_stag |
325 | * @post_sq: flag to post sq |
326 | */ |
327 | int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, |
328 | bool inv_stag, bool post_sq) |
329 | { |
330 | struct irdma_rdma_read *op_info; |
331 | int ret_code; |
332 | u32 i, byte_off, total_size = 0; |
333 | bool local_fence = false; |
334 | u32 addl_frag_cnt; |
335 | __le64 *wqe; |
336 | u32 wqe_idx; |
337 | u16 quanta; |
338 | u64 hdr; |
339 | |
340 | op_info = &info->op.rdma_read; |
341 | if (qp->max_sq_frag_cnt < op_info->num_lo_sges) |
342 | return -EINVAL; |
343 | |
344 | for (i = 0; i < op_info->num_lo_sges; i++) |
345 | total_size += op_info->lo_sg_list[i].length; |
346 | |
347 | ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt: op_info->num_lo_sges, quanta: &quanta); |
348 | if (ret_code) |
349 | return ret_code; |
350 | |
351 | wqe = irdma_qp_get_next_send_wqe(qp, wqe_idx: &wqe_idx, quanta, total_size, |
352 | info); |
353 | if (!wqe) |
354 | return -ENOMEM; |
355 | |
356 | irdma_clr_wqes(qp, qp_wqe_idx: wqe_idx); |
357 | |
358 | addl_frag_cnt = op_info->num_lo_sges > 1 ? |
359 | (op_info->num_lo_sges - 1) : 0; |
360 | local_fence |= info->local_fence; |
361 | |
362 | qp->wqe_ops.iw_set_fragment(wqe, 0, op_info->lo_sg_list, |
363 | qp->swqe_polarity); |
364 | for (i = 1, byte_off = 32; i < op_info->num_lo_sges; ++i) { |
365 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, |
366 | &op_info->lo_sg_list[i], |
367 | qp->swqe_polarity); |
368 | byte_off += 16; |
369 | } |
370 | |
371 | /* if not an odd number set valid bit in next fragment */ |
372 | if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && |
373 | !(op_info->num_lo_sges & 0x01) && op_info->num_lo_sges) { |
374 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, |
375 | qp->swqe_polarity); |
376 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) |
377 | ++addl_frag_cnt; |
378 | } |
379 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
380 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); |
381 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | |
382 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | |
383 | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | |
384 | FIELD_PREP(IRDMAQPSQ_OPCODE, |
385 | (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) | |
386 | FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | |
387 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) | |
388 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | |
389 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); |
390 | |
391 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ |
392 | |
393 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
394 | |
395 | if (post_sq) |
396 | irdma_uk_qp_post_wr(qp); |
397 | |
398 | return 0; |
399 | } |
400 | |
401 | /** |
402 | * irdma_uk_send - rdma send command |
403 | * @qp: hw qp ptr |
404 | * @info: post sq information |
405 | * @post_sq: flag to post sq |
406 | */ |
407 | int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, |
408 | bool post_sq) |
409 | { |
410 | __le64 *wqe; |
411 | struct irdma_post_send *op_info; |
412 | u64 hdr; |
413 | u32 i, wqe_idx, total_size = 0, byte_off; |
414 | int ret_code; |
415 | u32 frag_cnt, addl_frag_cnt; |
416 | bool read_fence = false; |
417 | u16 quanta; |
418 | |
419 | op_info = &info->op.send; |
420 | if (qp->max_sq_frag_cnt < op_info->num_sges) |
421 | return -EINVAL; |
422 | |
423 | for (i = 0; i < op_info->num_sges; i++) |
424 | total_size += op_info->sg_list[i].length; |
425 | |
426 | if (info->imm_data_valid) |
427 | frag_cnt = op_info->num_sges + 1; |
428 | else |
429 | frag_cnt = op_info->num_sges; |
430 | ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, quanta: &quanta); |
431 | if (ret_code) |
432 | return ret_code; |
433 | |
434 | wqe = irdma_qp_get_next_send_wqe(qp, wqe_idx: &wqe_idx, quanta, total_size, |
435 | info); |
436 | if (!wqe) |
437 | return -ENOMEM; |
438 | |
439 | irdma_clr_wqes(qp, qp_wqe_idx: wqe_idx); |
440 | |
441 | read_fence |= info->read_fence; |
442 | addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; |
443 | if (info->imm_data_valid) { |
444 | set_64bit_val(wqe_words: wqe, byte_index: 0, |
445 | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); |
446 | i = 0; |
447 | } else { |
448 | qp->wqe_ops.iw_set_fragment(wqe, 0, |
449 | frag_cnt ? op_info->sg_list : NULL, |
450 | qp->swqe_polarity); |
451 | i = 1; |
452 | } |
453 | |
454 | for (byte_off = 32; i < op_info->num_sges; i++) { |
455 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, &op_info->sg_list[i], |
456 | qp->swqe_polarity); |
457 | byte_off += 16; |
458 | } |
459 | |
460 | /* if not an odd number set valid bit in next fragment */ |
461 | if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) && |
462 | frag_cnt) { |
463 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, |
464 | qp->swqe_polarity); |
465 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) |
466 | ++addl_frag_cnt; |
467 | } |
468 | |
469 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
470 | FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) | |
471 | FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp)); |
472 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) | |
473 | FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) | |
474 | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, |
475 | (info->imm_data_valid ? 1 : 0)) | |
476 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | |
477 | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | |
478 | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | |
479 | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | |
480 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | |
481 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | |
482 | FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) | |
483 | FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) | |
484 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); |
485 | |
486 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ |
487 | |
488 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
489 | |
490 | if (post_sq) |
491 | irdma_uk_qp_post_wr(qp); |
492 | |
493 | return 0; |
494 | } |
495 | |
496 | /** |
497 | * irdma_set_mw_bind_wqe_gen_1 - set mw bind wqe |
498 | * @wqe: wqe for setting fragment |
499 | * @op_info: info for setting bind wqe values |
500 | */ |
501 | static void irdma_set_mw_bind_wqe_gen_1(__le64 *wqe, |
502 | struct irdma_bind_window *op_info) |
503 | { |
504 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: (uintptr_t)op_info->va); |
505 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
506 | FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) | |
507 | FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag)); |
508 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: op_info->bind_len); |
509 | } |
510 | |
511 | /** |
512 | * irdma_copy_inline_data_gen_1 - Copy inline data to wqe |
513 | * @wqe: pointer to wqe |
514 | * @sge_list: table of pointers to inline data |
515 | * @num_sges: Total inline data length |
516 | * @polarity: compatibility parameter |
517 | */ |
518 | static void irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list, |
519 | u32 num_sges, u8 polarity) |
520 | { |
521 | u32 quanta_bytes_remaining = 16; |
522 | int i; |
523 | |
524 | for (i = 0; i < num_sges; i++) { |
525 | u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr; |
526 | u32 sge_len = sge_list[i].length; |
527 | |
528 | while (sge_len) { |
529 | u32 bytes_copied; |
530 | |
531 | bytes_copied = min(sge_len, quanta_bytes_remaining); |
532 | memcpy(wqe, cur_sge, bytes_copied); |
533 | wqe += bytes_copied; |
534 | cur_sge += bytes_copied; |
535 | quanta_bytes_remaining -= bytes_copied; |
536 | sge_len -= bytes_copied; |
537 | |
538 | if (!quanta_bytes_remaining) { |
539 | /* Remaining inline bytes reside after hdr */ |
540 | wqe += 16; |
541 | quanta_bytes_remaining = 32; |
542 | } |
543 | } |
544 | } |
545 | } |
546 | |
547 | /** |
548 | * irdma_inline_data_size_to_quanta_gen_1 - based on inline data, quanta |
549 | * @data_size: data size for inline |
550 | * |
551 | * Gets the quanta based on inline and immediate data. |
552 | */ |
553 | static inline u16 irdma_inline_data_size_to_quanta_gen_1(u32 data_size) |
554 | { |
555 | return data_size <= 16 ? IRDMA_QP_WQE_MIN_QUANTA : 2; |
556 | } |
557 | |
558 | /** |
559 | * irdma_set_mw_bind_wqe - set mw bind in wqe |
560 | * @wqe: wqe for setting mw bind |
561 | * @op_info: info for setting wqe values |
562 | */ |
563 | static void irdma_set_mw_bind_wqe(__le64 *wqe, |
564 | struct irdma_bind_window *op_info) |
565 | { |
566 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: (uintptr_t)op_info->va); |
567 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
568 | FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) | |
569 | FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag)); |
570 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: op_info->bind_len); |
571 | } |
572 | |
573 | /** |
574 | * irdma_copy_inline_data - Copy inline data to wqe |
575 | * @wqe: pointer to wqe |
576 | * @sge_list: table of pointers to inline data |
577 | * @num_sges: number of SGE's |
578 | * @polarity: polarity of wqe valid bit |
579 | */ |
580 | static void irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list, |
581 | u32 num_sges, u8 polarity) |
582 | { |
583 | u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; |
584 | u32 quanta_bytes_remaining = 8; |
585 | bool first_quanta = true; |
586 | int i; |
587 | |
588 | wqe += 8; |
589 | |
590 | for (i = 0; i < num_sges; i++) { |
591 | u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr; |
592 | u32 sge_len = sge_list[i].length; |
593 | |
594 | while (sge_len) { |
595 | u32 bytes_copied; |
596 | |
597 | bytes_copied = min(sge_len, quanta_bytes_remaining); |
598 | memcpy(wqe, cur_sge, bytes_copied); |
599 | wqe += bytes_copied; |
600 | cur_sge += bytes_copied; |
601 | quanta_bytes_remaining -= bytes_copied; |
602 | sge_len -= bytes_copied; |
603 | |
604 | if (!quanta_bytes_remaining) { |
605 | quanta_bytes_remaining = 31; |
606 | |
607 | /* Remaining inline bytes reside after hdr */ |
608 | if (first_quanta) { |
609 | first_quanta = false; |
610 | wqe += 16; |
611 | } else { |
612 | *wqe = inline_valid; |
613 | wqe++; |
614 | } |
615 | } |
616 | } |
617 | } |
618 | if (!first_quanta && quanta_bytes_remaining < 31) |
619 | *(wqe + quanta_bytes_remaining) = inline_valid; |
620 | } |
621 | |
622 | /** |
623 | * irdma_inline_data_size_to_quanta - based on inline data, quanta |
624 | * @data_size: data size for inline |
625 | * |
626 | * Gets the quanta based on inline and immediate data. |
627 | */ |
628 | static u16 irdma_inline_data_size_to_quanta(u32 data_size) |
629 | { |
630 | if (data_size <= 8) |
631 | return IRDMA_QP_WQE_MIN_QUANTA; |
632 | else if (data_size <= 39) |
633 | return 2; |
634 | else if (data_size <= 70) |
635 | return 3; |
636 | else if (data_size <= 101) |
637 | return 4; |
638 | else if (data_size <= 132) |
639 | return 5; |
640 | else if (data_size <= 163) |
641 | return 6; |
642 | else if (data_size <= 194) |
643 | return 7; |
644 | else |
645 | return 8; |
646 | } |
647 | |
648 | /** |
649 | * irdma_uk_inline_rdma_write - inline rdma write operation |
650 | * @qp: hw qp ptr |
651 | * @info: post sq information |
652 | * @post_sq: flag to post sq |
653 | */ |
654 | int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, |
655 | struct irdma_post_sq_info *info, bool post_sq) |
656 | { |
657 | __le64 *wqe; |
658 | struct irdma_rdma_write *op_info; |
659 | u64 hdr = 0; |
660 | u32 wqe_idx; |
661 | bool read_fence = false; |
662 | u32 i, total_size = 0; |
663 | u16 quanta; |
664 | |
665 | op_info = &info->op.rdma_write; |
666 | |
667 | if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges)) |
668 | return -EINVAL; |
669 | |
670 | for (i = 0; i < op_info->num_lo_sges; i++) |
671 | total_size += op_info->lo_sg_list[i].length; |
672 | |
673 | if (unlikely(total_size > qp->max_inline_data)) |
674 | return -EINVAL; |
675 | |
676 | quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); |
677 | wqe = irdma_qp_get_next_send_wqe(qp, wqe_idx: &wqe_idx, quanta, total_size, |
678 | info); |
679 | if (!wqe) |
680 | return -ENOMEM; |
681 | |
682 | irdma_clr_wqes(qp, qp_wqe_idx: wqe_idx); |
683 | |
684 | read_fence |= info->read_fence; |
685 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
686 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); |
687 | |
688 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | |
689 | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | |
690 | FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) | |
691 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) | |
692 | FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) | |
693 | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) | |
694 | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | |
695 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | |
696 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | |
697 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); |
698 | |
699 | if (info->imm_data_valid) |
700 | set_64bit_val(wqe_words: wqe, byte_index: 0, |
701 | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); |
702 | |
703 | qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list, |
704 | op_info->num_lo_sges, |
705 | qp->swqe_polarity); |
706 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ |
707 | |
708 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
709 | |
710 | if (post_sq) |
711 | irdma_uk_qp_post_wr(qp); |
712 | |
713 | return 0; |
714 | } |
715 | |
716 | /** |
717 | * irdma_uk_inline_send - inline send operation |
718 | * @qp: hw qp ptr |
719 | * @info: post sq information |
720 | * @post_sq: flag to post sq |
721 | */ |
722 | int irdma_uk_inline_send(struct irdma_qp_uk *qp, |
723 | struct irdma_post_sq_info *info, bool post_sq) |
724 | { |
725 | __le64 *wqe; |
726 | struct irdma_post_send *op_info; |
727 | u64 hdr; |
728 | u32 wqe_idx; |
729 | bool read_fence = false; |
730 | u32 i, total_size = 0; |
731 | u16 quanta; |
732 | |
733 | op_info = &info->op.send; |
734 | |
735 | if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges)) |
736 | return -EINVAL; |
737 | |
738 | for (i = 0; i < op_info->num_sges; i++) |
739 | total_size += op_info->sg_list[i].length; |
740 | |
741 | if (unlikely(total_size > qp->max_inline_data)) |
742 | return -EINVAL; |
743 | |
744 | quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); |
745 | wqe = irdma_qp_get_next_send_wqe(qp, wqe_idx: &wqe_idx, quanta, total_size, |
746 | info); |
747 | if (!wqe) |
748 | return -ENOMEM; |
749 | |
750 | irdma_clr_wqes(qp, qp_wqe_idx: wqe_idx); |
751 | |
752 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
753 | FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) | |
754 | FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp)); |
755 | |
756 | read_fence |= info->read_fence; |
757 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) | |
758 | FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) | |
759 | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | |
760 | FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) | |
761 | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, |
762 | (info->imm_data_valid ? 1 : 0)) | |
763 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | |
764 | FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) | |
765 | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | |
766 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | |
767 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | |
768 | FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) | |
769 | FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) | |
770 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); |
771 | |
772 | if (info->imm_data_valid) |
773 | set_64bit_val(wqe_words: wqe, byte_index: 0, |
774 | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); |
775 | qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list, |
776 | op_info->num_sges, qp->swqe_polarity); |
777 | |
778 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ |
779 | |
780 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
781 | |
782 | if (post_sq) |
783 | irdma_uk_qp_post_wr(qp); |
784 | |
785 | return 0; |
786 | } |
787 | |
788 | /** |
789 | * irdma_uk_stag_local_invalidate - stag invalidate operation |
790 | * @qp: hw qp ptr |
791 | * @info: post sq information |
792 | * @post_sq: flag to post sq |
793 | */ |
794 | int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, |
795 | struct irdma_post_sq_info *info, |
796 | bool post_sq) |
797 | { |
798 | __le64 *wqe; |
799 | struct irdma_inv_local_stag *op_info; |
800 | u64 hdr; |
801 | u32 wqe_idx; |
802 | bool local_fence = false; |
803 | struct ib_sge sge = {}; |
804 | |
805 | op_info = &info->op.inv_local_stag; |
806 | local_fence = info->local_fence; |
807 | |
808 | wqe = irdma_qp_get_next_send_wqe(qp, wqe_idx: &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA, |
809 | total_size: 0, info); |
810 | if (!wqe) |
811 | return -ENOMEM; |
812 | |
813 | irdma_clr_wqes(qp, qp_wqe_idx: wqe_idx); |
814 | |
815 | sge.lkey = op_info->target_stag; |
816 | qp->wqe_ops.iw_set_fragment(wqe, 0, &sge, 0); |
817 | |
818 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: 0); |
819 | |
820 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_INV_STAG) | |
821 | FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | |
822 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) | |
823 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | |
824 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); |
825 | |
826 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ |
827 | |
828 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
829 | |
830 | if (post_sq) |
831 | irdma_uk_qp_post_wr(qp); |
832 | |
833 | return 0; |
834 | } |
835 | |
836 | /** |
837 | * irdma_uk_post_receive - post receive wqe |
838 | * @qp: hw qp ptr |
839 | * @info: post rq information |
840 | */ |
841 | int irdma_uk_post_receive(struct irdma_qp_uk *qp, |
842 | struct irdma_post_rq_info *info) |
843 | { |
844 | u32 wqe_idx, i, byte_off; |
845 | u32 addl_frag_cnt; |
846 | __le64 *wqe; |
847 | u64 hdr; |
848 | |
849 | if (qp->max_rq_frag_cnt < info->num_sges) |
850 | return -EINVAL; |
851 | |
852 | wqe = irdma_qp_get_next_recv_wqe(qp, wqe_idx: &wqe_idx); |
853 | if (!wqe) |
854 | return -ENOMEM; |
855 | |
856 | qp->rq_wrid_array[wqe_idx] = info->wr_id; |
857 | addl_frag_cnt = info->num_sges > 1 ? (info->num_sges - 1) : 0; |
858 | qp->wqe_ops.iw_set_fragment(wqe, 0, info->sg_list, |
859 | qp->rwqe_polarity); |
860 | |
861 | for (i = 1, byte_off = 32; i < info->num_sges; i++) { |
862 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i], |
863 | qp->rwqe_polarity); |
864 | byte_off += 16; |
865 | } |
866 | |
867 | /* if not an odd number set valid bit in next fragment */ |
868 | if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) && |
869 | info->num_sges) { |
870 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, |
871 | qp->rwqe_polarity); |
872 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) |
873 | ++addl_frag_cnt; |
874 | } |
875 | |
876 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: 0); |
877 | hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | |
878 | FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity); |
879 | |
880 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ |
881 | |
882 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
883 | |
884 | return 0; |
885 | } |
886 | |
887 | /** |
888 | * irdma_uk_cq_resize - reset the cq buffer info |
889 | * @cq: cq to resize |
890 | * @cq_base: new cq buffer addr |
891 | * @cq_size: number of cqes |
892 | */ |
893 | void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int cq_size) |
894 | { |
895 | cq->cq_base = cq_base; |
896 | cq->cq_size = cq_size; |
897 | IRDMA_RING_INIT(cq->cq_ring, cq->cq_size); |
898 | cq->polarity = 1; |
899 | } |
900 | |
901 | /** |
902 | * irdma_uk_cq_set_resized_cnt - record the count of the resized buffers |
903 | * @cq: cq to resize |
904 | * @cq_cnt: the count of the resized cq buffers |
905 | */ |
906 | void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *cq, u16 cq_cnt) |
907 | { |
908 | u64 temp_val; |
909 | u16 sw_cq_sel; |
910 | u8 arm_next_se; |
911 | u8 arm_next; |
912 | u8 arm_seq_num; |
913 | |
914 | get_64bit_val(wqe_words: cq->shadow_area, byte_index: 32, val: &temp_val); |
915 | |
916 | sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); |
917 | sw_cq_sel += cq_cnt; |
918 | |
919 | arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); |
920 | arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); |
921 | arm_next = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT, temp_val); |
922 | |
923 | temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) | |
924 | FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) | |
925 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | |
926 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next); |
927 | |
928 | set_64bit_val(wqe_words: cq->shadow_area, byte_index: 32, val: temp_val); |
929 | } |
930 | |
931 | /** |
932 | * irdma_uk_cq_request_notification - cq notification request (door bell) |
933 | * @cq: hw cq |
934 | * @cq_notify: notification type |
935 | */ |
936 | void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq, |
937 | enum irdma_cmpl_notify cq_notify) |
938 | { |
939 | u64 temp_val; |
940 | u16 sw_cq_sel; |
941 | u8 arm_next_se = 0; |
942 | u8 arm_next = 0; |
943 | u8 arm_seq_num; |
944 | |
945 | get_64bit_val(wqe_words: cq->shadow_area, byte_index: 32, val: &temp_val); |
946 | arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); |
947 | arm_seq_num++; |
948 | sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); |
949 | arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); |
950 | arm_next_se |= 1; |
951 | if (cq_notify == IRDMA_CQ_COMPL_EVENT) |
952 | arm_next = 1; |
953 | temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) | |
954 | FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) | |
955 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | |
956 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next); |
957 | |
958 | set_64bit_val(wqe_words: cq->shadow_area, byte_index: 32, val: temp_val); |
959 | |
960 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ |
961 | |
962 | writel(val: cq->cq_id, addr: cq->cqe_alloc_db); |
963 | } |
964 | |
965 | /** |
966 | * irdma_uk_cq_poll_cmpl - get cq completion info |
967 | * @cq: hw cq |
968 | * @info: cq poll information returned |
969 | */ |
970 | int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, |
971 | struct irdma_cq_poll_info *info) |
972 | { |
973 | u64 comp_ctx, qword0, qword2, qword3; |
974 | __le64 *cqe; |
975 | struct irdma_qp_uk *qp; |
976 | struct irdma_ring *pring = NULL; |
977 | u32 wqe_idx; |
978 | int ret_code; |
979 | bool move_cq_head = true; |
980 | u8 polarity; |
981 | bool ext_valid; |
982 | __le64 *ext_cqe; |
983 | |
984 | if (cq->avoid_mem_cflct) |
985 | cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(cq); |
986 | else |
987 | cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq); |
988 | |
989 | get_64bit_val(wqe_words: cqe, byte_index: 24, val: &qword3); |
990 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); |
991 | if (polarity != cq->polarity) |
992 | return -ENOENT; |
993 | |
994 | /* Ensure CQE contents are read after valid bit is checked */ |
995 | dma_rmb(); |
996 | |
997 | ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3); |
998 | if (ext_valid) { |
999 | u64 qword6, qword7; |
1000 | u32 peek_head; |
1001 | |
1002 | if (cq->avoid_mem_cflct) { |
1003 | ext_cqe = (__le64 *)((u8 *)cqe + 32); |
1004 | get_64bit_val(wqe_words: ext_cqe, byte_index: 24, val: &qword7); |
1005 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7); |
1006 | } else { |
1007 | peek_head = (cq->cq_ring.head + 1) % cq->cq_ring.size; |
1008 | ext_cqe = cq->cq_base[peek_head].buf; |
1009 | get_64bit_val(wqe_words: ext_cqe, byte_index: 24, val: &qword7); |
1010 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7); |
1011 | if (!peek_head) |
1012 | polarity ^= 1; |
1013 | } |
1014 | if (polarity != cq->polarity) |
1015 | return -ENOENT; |
1016 | |
1017 | /* Ensure ext CQE contents are read after ext valid bit is checked */ |
1018 | dma_rmb(); |
1019 | |
1020 | info->imm_valid = (bool)FIELD_GET(IRDMA_CQ_IMMVALID, qword7); |
1021 | if (info->imm_valid) { |
1022 | u64 qword4; |
1023 | |
1024 | get_64bit_val(wqe_words: ext_cqe, byte_index: 0, val: &qword4); |
1025 | info->imm_data = (u32)FIELD_GET(IRDMA_CQ_IMMDATALOW32, qword4); |
1026 | } |
1027 | info->ud_smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7); |
1028 | info->ud_vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7); |
1029 | if (info->ud_smac_valid || info->ud_vlan_valid) { |
1030 | get_64bit_val(wqe_words: ext_cqe, byte_index: 16, val: &qword6); |
1031 | if (info->ud_vlan_valid) |
1032 | info->ud_vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6); |
1033 | if (info->ud_smac_valid) { |
1034 | info->ud_smac[5] = qword6 & 0xFF; |
1035 | info->ud_smac[4] = (qword6 >> 8) & 0xFF; |
1036 | info->ud_smac[3] = (qword6 >> 16) & 0xFF; |
1037 | info->ud_smac[2] = (qword6 >> 24) & 0xFF; |
1038 | info->ud_smac[1] = (qword6 >> 32) & 0xFF; |
1039 | info->ud_smac[0] = (qword6 >> 40) & 0xFF; |
1040 | } |
1041 | } |
1042 | } else { |
1043 | info->imm_valid = false; |
1044 | info->ud_smac_valid = false; |
1045 | info->ud_vlan_valid = false; |
1046 | } |
1047 | |
1048 | info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3); |
1049 | info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3); |
1050 | info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3); |
1051 | if (info->error) { |
1052 | info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3); |
1053 | info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3); |
1054 | if (info->major_err == IRDMA_FLUSH_MAJOR_ERR) { |
1055 | info->comp_status = IRDMA_COMPL_STATUS_FLUSHED; |
1056 | /* Set the min error to standard flush error code for remaining cqes */ |
1057 | if (info->minor_err != FLUSH_GENERAL_ERR) { |
1058 | qword3 &= ~IRDMA_CQ_MINERR; |
1059 | qword3 |= FIELD_PREP(IRDMA_CQ_MINERR, FLUSH_GENERAL_ERR); |
1060 | set_64bit_val(wqe_words: cqe, byte_index: 24, val: qword3); |
1061 | } |
1062 | } else { |
1063 | info->comp_status = IRDMA_COMPL_STATUS_UNKNOWN; |
1064 | } |
1065 | } else { |
1066 | info->comp_status = IRDMA_COMPL_STATUS_SUCCESS; |
1067 | } |
1068 | |
1069 | get_64bit_val(wqe_words: cqe, byte_index: 0, val: &qword0); |
1070 | get_64bit_val(wqe_words: cqe, byte_index: 16, val: &qword2); |
1071 | |
1072 | info->tcp_seq_num_rtt = (u32)FIELD_GET(IRDMACQ_TCPSEQNUMRTT, qword0); |
1073 | info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2); |
1074 | info->ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2); |
1075 | |
1076 | get_64bit_val(wqe_words: cqe, byte_index: 8, val: &comp_ctx); |
1077 | |
1078 | info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3); |
1079 | qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx; |
1080 | if (!qp || qp->destroy_pending) { |
1081 | ret_code = -EFAULT; |
1082 | goto exit; |
1083 | } |
1084 | wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3); |
1085 | info->qp_handle = (irdma_qp_handle)(unsigned long)qp; |
1086 | info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3); |
1087 | |
1088 | if (info->q_type == IRDMA_CQE_QTYPE_RQ) { |
1089 | u32 array_idx; |
1090 | |
1091 | array_idx = wqe_idx / qp->rq_wqe_size_multiplier; |
1092 | |
1093 | if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED || |
1094 | info->comp_status == IRDMA_COMPL_STATUS_UNKNOWN) { |
1095 | if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) { |
1096 | ret_code = -ENOENT; |
1097 | goto exit; |
1098 | } |
1099 | |
1100 | info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail]; |
1101 | array_idx = qp->rq_ring.tail; |
1102 | } else { |
1103 | info->wr_id = qp->rq_wrid_array[array_idx]; |
1104 | } |
1105 | |
1106 | info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0); |
1107 | |
1108 | if (qword3 & IRDMACQ_STAG) { |
1109 | info->stag_invalid_set = true; |
1110 | info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2); |
1111 | } else { |
1112 | info->stag_invalid_set = false; |
1113 | } |
1114 | IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1); |
1115 | if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) { |
1116 | qp->rq_flush_seen = true; |
1117 | if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) |
1118 | qp->rq_flush_complete = true; |
1119 | else |
1120 | move_cq_head = false; |
1121 | } |
1122 | pring = &qp->rq_ring; |
1123 | } else { /* q_type is IRDMA_CQE_QTYPE_SQ */ |
1124 | if (qp->first_sq_wq) { |
1125 | if (wqe_idx + 1 >= qp->conn_wqes) |
1126 | qp->first_sq_wq = false; |
1127 | |
1128 | if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) { |
1129 | IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); |
1130 | IRDMA_RING_MOVE_TAIL(cq->cq_ring); |
1131 | set_64bit_val(wqe_words: cq->shadow_area, byte_index: 0, |
1132 | IRDMA_RING_CURRENT_HEAD(cq->cq_ring)); |
1133 | memset(info, 0, |
1134 | sizeof(struct irdma_cq_poll_info)); |
1135 | return irdma_uk_cq_poll_cmpl(cq, info); |
1136 | } |
1137 | } |
1138 | if (info->comp_status != IRDMA_COMPL_STATUS_FLUSHED) { |
1139 | info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; |
1140 | if (!info->comp_status) |
1141 | info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; |
1142 | info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3); |
1143 | IRDMA_RING_SET_TAIL(qp->sq_ring, |
1144 | wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); |
1145 | } else { |
1146 | if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) { |
1147 | ret_code = -ENOENT; |
1148 | goto exit; |
1149 | } |
1150 | |
1151 | do { |
1152 | __le64 *sw_wqe; |
1153 | u64 wqe_qword; |
1154 | u32 tail; |
1155 | |
1156 | tail = qp->sq_ring.tail; |
1157 | sw_wqe = qp->sq_base[tail].elem; |
1158 | get_64bit_val(wqe_words: sw_wqe, byte_index: 24, |
1159 | val: &wqe_qword); |
1160 | info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, |
1161 | wqe_qword); |
1162 | IRDMA_RING_SET_TAIL(qp->sq_ring, |
1163 | tail + qp->sq_wrtrk_array[tail].quanta); |
1164 | if (info->op_type != IRDMAQP_OP_NOP) { |
1165 | info->wr_id = qp->sq_wrtrk_array[tail].wrid; |
1166 | info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len; |
1167 | break; |
1168 | } |
1169 | } while (1); |
1170 | if (info->op_type == IRDMA_OP_TYPE_BIND_MW && |
1171 | info->minor_err == FLUSH_PROT_ERR) |
1172 | info->minor_err = FLUSH_MW_BIND_ERR; |
1173 | qp->sq_flush_seen = true; |
1174 | if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) |
1175 | qp->sq_flush_complete = true; |
1176 | } |
1177 | pring = &qp->sq_ring; |
1178 | } |
1179 | |
1180 | ret_code = 0; |
1181 | |
1182 | exit: |
1183 | if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) |
1184 | if (pring && IRDMA_RING_MORE_WORK(*pring)) |
1185 | move_cq_head = false; |
1186 | |
1187 | if (move_cq_head) { |
1188 | IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); |
1189 | if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring)) |
1190 | cq->polarity ^= 1; |
1191 | |
1192 | if (ext_valid && !cq->avoid_mem_cflct) { |
1193 | IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); |
1194 | if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring)) |
1195 | cq->polarity ^= 1; |
1196 | } |
1197 | |
1198 | IRDMA_RING_MOVE_TAIL(cq->cq_ring); |
1199 | if (!cq->avoid_mem_cflct && ext_valid) |
1200 | IRDMA_RING_MOVE_TAIL(cq->cq_ring); |
1201 | set_64bit_val(wqe_words: cq->shadow_area, byte_index: 0, |
1202 | IRDMA_RING_CURRENT_HEAD(cq->cq_ring)); |
1203 | } else { |
1204 | qword3 &= ~IRDMA_CQ_WQEIDX; |
1205 | qword3 |= FIELD_PREP(IRDMA_CQ_WQEIDX, pring->tail); |
1206 | set_64bit_val(wqe_words: cqe, byte_index: 24, val: qword3); |
1207 | } |
1208 | |
1209 | return ret_code; |
1210 | } |
1211 | |
1212 | /** |
1213 | * irdma_qp_round_up - return round up qp wq depth |
1214 | * @wqdepth: wq depth in quanta to round up |
1215 | */ |
1216 | static int irdma_qp_round_up(u32 wqdepth) |
1217 | { |
1218 | int scount = 1; |
1219 | |
1220 | for (wqdepth--; scount <= 16; scount *= 2) |
1221 | wqdepth |= wqdepth >> scount; |
1222 | |
1223 | return ++wqdepth; |
1224 | } |
1225 | |
1226 | /** |
1227 | * irdma_get_wqe_shift - get shift count for maximum wqe size |
1228 | * @uk_attrs: qp HW attributes |
1229 | * @sge: Maximum Scatter Gather Elements wqe |
1230 | * @inline_data: Maximum inline data size |
1231 | * @shift: Returns the shift needed based on sge |
1232 | * |
1233 | * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size. |
1234 | * For 1 SGE or inline data <= 8, shift = 0 (wqe size of 32 |
1235 | * bytes). For 2 or 3 SGEs or inline data <= 39, shift = 1 (wqe |
1236 | * size of 64 bytes). |
1237 | * For 4-7 SGE's and inline <= 101 Shift of 2 otherwise (wqe |
1238 | * size of 256 bytes). |
1239 | */ |
1240 | void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, |
1241 | u32 inline_data, u8 *shift) |
1242 | { |
1243 | *shift = 0; |
1244 | if (uk_attrs->hw_rev >= IRDMA_GEN_2) { |
1245 | if (sge > 1 || inline_data > 8) { |
1246 | if (sge < 4 && inline_data <= 39) |
1247 | *shift = 1; |
1248 | else if (sge < 8 && inline_data <= 101) |
1249 | *shift = 2; |
1250 | else |
1251 | *shift = 3; |
1252 | } |
1253 | } else if (sge > 1 || inline_data > 16) { |
1254 | *shift = (sge < 4 && inline_data <= 48) ? 1 : 2; |
1255 | } |
1256 | } |
1257 | |
1258 | /* |
1259 | * irdma_get_sqdepth - get SQ depth (quanta) |
1260 | * @uk_attrs: qp HW attributes |
1261 | * @sq_size: SQ size |
1262 | * @shift: shift which determines size of WQE |
1263 | * @sqdepth: depth of SQ |
1264 | * |
1265 | */ |
1266 | int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, |
1267 | u32 *sqdepth) |
1268 | { |
1269 | u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; |
1270 | |
1271 | *sqdepth = irdma_qp_round_up(wqdepth: (sq_size << shift) + IRDMA_SQ_RSVD); |
1272 | |
1273 | if (*sqdepth < min_size) |
1274 | *sqdepth = min_size; |
1275 | else if (*sqdepth > uk_attrs->max_hw_wq_quanta) |
1276 | return -EINVAL; |
1277 | |
1278 | return 0; |
1279 | } |
1280 | |
1281 | /* |
1282 | * irdma_get_rqdepth - get RQ depth (quanta) |
1283 | * @uk_attrs: qp HW attributes |
1284 | * @rq_size: RQ size |
1285 | * @shift: shift which determines size of WQE |
1286 | * @rqdepth: depth of RQ |
1287 | */ |
1288 | int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, |
1289 | u32 *rqdepth) |
1290 | { |
1291 | u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; |
1292 | |
1293 | *rqdepth = irdma_qp_round_up(wqdepth: (rq_size << shift) + IRDMA_RQ_RSVD); |
1294 | |
1295 | if (*rqdepth < min_size) |
1296 | *rqdepth = min_size; |
1297 | else if (*rqdepth > uk_attrs->max_hw_rq_quanta) |
1298 | return -EINVAL; |
1299 | |
1300 | return 0; |
1301 | } |
1302 | |
1303 | static const struct irdma_wqe_uk_ops iw_wqe_uk_ops = { |
1304 | .iw_copy_inline_data = irdma_copy_inline_data, |
1305 | .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta, |
1306 | .iw_set_fragment = irdma_set_fragment, |
1307 | .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe, |
1308 | }; |
1309 | |
1310 | static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = { |
1311 | .iw_copy_inline_data = irdma_copy_inline_data_gen_1, |
1312 | .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta_gen_1, |
1313 | .iw_set_fragment = irdma_set_fragment_gen_1, |
1314 | .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe_gen_1, |
1315 | }; |
1316 | |
1317 | /** |
1318 | * irdma_setup_connection_wqes - setup WQEs necessary to complete |
1319 | * connection. |
1320 | * @qp: hw qp (user and kernel) |
1321 | * @info: qp initialization info |
1322 | */ |
1323 | static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp, |
1324 | struct irdma_qp_uk_init_info *info) |
1325 | { |
1326 | u16 move_cnt = 1; |
1327 | |
1328 | if (!info->legacy_mode && |
1329 | (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)) |
1330 | move_cnt = 3; |
1331 | |
1332 | qp->conn_wqes = move_cnt; |
1333 | IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt); |
1334 | IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt); |
1335 | IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt); |
1336 | } |
1337 | |
1338 | /** |
1339 | * irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ |
1340 | * @ukinfo: qp initialization info |
1341 | * @sq_shift: Returns shift of SQ |
1342 | * @rq_shift: Returns shift of RQ |
1343 | */ |
1344 | void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift, |
1345 | u8 *rq_shift) |
1346 | { |
1347 | bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2; |
1348 | |
1349 | irdma_get_wqe_shift(uk_attrs: ukinfo->uk_attrs, |
1350 | sge: imm_support ? ukinfo->max_sq_frag_cnt + 1 : |
1351 | ukinfo->max_sq_frag_cnt, |
1352 | inline_data: ukinfo->max_inline_data, shift: sq_shift); |
1353 | |
1354 | irdma_get_wqe_shift(uk_attrs: ukinfo->uk_attrs, sge: ukinfo->max_rq_frag_cnt, inline_data: 0, |
1355 | shift: rq_shift); |
1356 | |
1357 | if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) { |
1358 | if (ukinfo->abi_ver > 4) |
1359 | *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; |
1360 | } |
1361 | } |
1362 | |
1363 | /** |
1364 | * irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size. |
1365 | * @ukinfo: qp initialization info |
1366 | * @sq_depth: Returns depth of SQ |
1367 | * @sq_shift: Returns shift of SQ |
1368 | */ |
1369 | int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo, |
1370 | u32 *sq_depth, u8 *sq_shift) |
1371 | { |
1372 | bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2; |
1373 | int status; |
1374 | |
1375 | irdma_get_wqe_shift(uk_attrs: ukinfo->uk_attrs, |
1376 | sge: imm_support ? ukinfo->max_sq_frag_cnt + 1 : |
1377 | ukinfo->max_sq_frag_cnt, |
1378 | inline_data: ukinfo->max_inline_data, shift: sq_shift); |
1379 | status = irdma_get_sqdepth(uk_attrs: ukinfo->uk_attrs, sq_size: ukinfo->sq_size, |
1380 | shift: *sq_shift, sqdepth: sq_depth); |
1381 | |
1382 | return status; |
1383 | } |
1384 | |
1385 | /** |
1386 | * irdma_uk_calc_depth_shift_rq - calculate depth and shift for RQ size. |
1387 | * @ukinfo: qp initialization info |
1388 | * @rq_depth: Returns depth of RQ |
1389 | * @rq_shift: Returns shift of RQ |
1390 | */ |
1391 | int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo, |
1392 | u32 *rq_depth, u8 *rq_shift) |
1393 | { |
1394 | int status; |
1395 | |
1396 | irdma_get_wqe_shift(uk_attrs: ukinfo->uk_attrs, sge: ukinfo->max_rq_frag_cnt, inline_data: 0, |
1397 | shift: rq_shift); |
1398 | |
1399 | if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) { |
1400 | if (ukinfo->abi_ver > 4) |
1401 | *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; |
1402 | } |
1403 | |
1404 | status = irdma_get_rqdepth(uk_attrs: ukinfo->uk_attrs, rq_size: ukinfo->rq_size, |
1405 | shift: *rq_shift, rqdepth: rq_depth); |
1406 | |
1407 | return status; |
1408 | } |
1409 | |
1410 | /** |
1411 | * irdma_uk_qp_init - initialize shared qp |
1412 | * @qp: hw qp (user and kernel) |
1413 | * @info: qp initialization info |
1414 | * |
1415 | * initializes the vars used in both user and kernel mode. |
1416 | * size of the wqe depends on numbers of max. fragements |
1417 | * allowed. Then size of wqe * the number of wqes should be the |
1418 | * amount of memory allocated for sq and rq. |
1419 | */ |
1420 | int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) |
1421 | { |
1422 | int ret_code = 0; |
1423 | u32 sq_ring_size; |
1424 | |
1425 | qp->uk_attrs = info->uk_attrs; |
1426 | if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags || |
1427 | info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags) |
1428 | return -EINVAL; |
1429 | |
1430 | qp->qp_caps = info->qp_caps; |
1431 | qp->sq_base = info->sq; |
1432 | qp->rq_base = info->rq; |
1433 | qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP; |
1434 | qp->shadow_area = info->shadow_area; |
1435 | qp->sq_wrtrk_array = info->sq_wrtrk_array; |
1436 | |
1437 | qp->rq_wrid_array = info->rq_wrid_array; |
1438 | qp->wqe_alloc_db = info->wqe_alloc_db; |
1439 | qp->qp_id = info->qp_id; |
1440 | qp->sq_size = info->sq_size; |
1441 | qp->max_sq_frag_cnt = info->max_sq_frag_cnt; |
1442 | sq_ring_size = qp->sq_size << info->sq_shift; |
1443 | IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); |
1444 | IRDMA_RING_INIT(qp->initial_ring, sq_ring_size); |
1445 | if (info->first_sq_wq) { |
1446 | irdma_setup_connection_wqes(qp, info); |
1447 | qp->swqe_polarity = 1; |
1448 | qp->first_sq_wq = true; |
1449 | } else { |
1450 | qp->swqe_polarity = 0; |
1451 | } |
1452 | qp->swqe_polarity_deferred = 1; |
1453 | qp->rwqe_polarity = 0; |
1454 | qp->rq_size = info->rq_size; |
1455 | qp->max_rq_frag_cnt = info->max_rq_frag_cnt; |
1456 | qp->max_inline_data = info->max_inline_data; |
1457 | qp->rq_wqe_size = info->rq_shift; |
1458 | IRDMA_RING_INIT(qp->rq_ring, qp->rq_size); |
1459 | qp->rq_wqe_size_multiplier = 1 << info->rq_shift; |
1460 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) |
1461 | qp->wqe_ops = iw_wqe_uk_ops_gen_1; |
1462 | else |
1463 | qp->wqe_ops = iw_wqe_uk_ops; |
1464 | return ret_code; |
1465 | } |
1466 | |
1467 | /** |
1468 | * irdma_uk_cq_init - initialize shared cq (user and kernel) |
1469 | * @cq: hw cq |
1470 | * @info: hw cq initialization info |
1471 | */ |
1472 | void irdma_uk_cq_init(struct irdma_cq_uk *cq, |
1473 | struct irdma_cq_uk_init_info *info) |
1474 | { |
1475 | cq->cq_base = info->cq_base; |
1476 | cq->cq_id = info->cq_id; |
1477 | cq->cq_size = info->cq_size; |
1478 | cq->cqe_alloc_db = info->cqe_alloc_db; |
1479 | cq->cq_ack_db = info->cq_ack_db; |
1480 | cq->shadow_area = info->shadow_area; |
1481 | cq->avoid_mem_cflct = info->avoid_mem_cflct; |
1482 | IRDMA_RING_INIT(cq->cq_ring, cq->cq_size); |
1483 | cq->polarity = 1; |
1484 | } |
1485 | |
1486 | /** |
1487 | * irdma_uk_clean_cq - clean cq entries |
1488 | * @q: completion context |
1489 | * @cq: cq to clean |
1490 | */ |
1491 | void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq) |
1492 | { |
1493 | __le64 *cqe; |
1494 | u64 qword3, comp_ctx; |
1495 | u32 cq_head; |
1496 | u8 polarity, temp; |
1497 | |
1498 | cq_head = cq->cq_ring.head; |
1499 | temp = cq->polarity; |
1500 | do { |
1501 | if (cq->avoid_mem_cflct) |
1502 | cqe = ((struct irdma_extended_cqe *)(cq->cq_base))[cq_head].buf; |
1503 | else |
1504 | cqe = cq->cq_base[cq_head].buf; |
1505 | get_64bit_val(wqe_words: cqe, byte_index: 24, val: &qword3); |
1506 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); |
1507 | |
1508 | if (polarity != temp) |
1509 | break; |
1510 | |
1511 | /* Ensure CQE contents are read after valid bit is checked */ |
1512 | dma_rmb(); |
1513 | |
1514 | get_64bit_val(wqe_words: cqe, byte_index: 8, val: &comp_ctx); |
1515 | if ((void *)(unsigned long)comp_ctx == q) |
1516 | set_64bit_val(wqe_words: cqe, byte_index: 8, val: 0); |
1517 | |
1518 | cq_head = (cq_head + 1) % cq->cq_ring.size; |
1519 | if (!cq_head) |
1520 | temp ^= 1; |
1521 | } while (true); |
1522 | } |
1523 | |
1524 | /** |
1525 | * irdma_nop - post a nop |
1526 | * @qp: hw qp ptr |
1527 | * @wr_id: work request id |
1528 | * @signaled: signaled for completion |
1529 | * @post_sq: ring doorbell |
1530 | */ |
1531 | int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq) |
1532 | { |
1533 | __le64 *wqe; |
1534 | u64 hdr; |
1535 | u32 wqe_idx; |
1536 | struct irdma_post_sq_info info = {}; |
1537 | |
1538 | info.wr_id = wr_id; |
1539 | wqe = irdma_qp_get_next_send_wqe(qp, wqe_idx: &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA, |
1540 | total_size: 0, info: &info); |
1541 | if (!wqe) |
1542 | return -ENOMEM; |
1543 | |
1544 | irdma_clr_wqes(qp, qp_wqe_idx: wqe_idx); |
1545 | |
1546 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: 0); |
1547 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: 0); |
1548 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: 0); |
1549 | |
1550 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | |
1551 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) | |
1552 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); |
1553 | |
1554 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ |
1555 | |
1556 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1557 | if (post_sq) |
1558 | irdma_uk_qp_post_wr(qp); |
1559 | |
1560 | return 0; |
1561 | } |
1562 | |
1563 | /** |
1564 | * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ |
1565 | * @frag_cnt: number of fragments |
1566 | * @quanta: quanta for frag_cnt |
1567 | */ |
1568 | int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta) |
1569 | { |
1570 | switch (frag_cnt) { |
1571 | case 0: |
1572 | case 1: |
1573 | *quanta = IRDMA_QP_WQE_MIN_QUANTA; |
1574 | break; |
1575 | case 2: |
1576 | case 3: |
1577 | *quanta = 2; |
1578 | break; |
1579 | case 4: |
1580 | case 5: |
1581 | *quanta = 3; |
1582 | break; |
1583 | case 6: |
1584 | case 7: |
1585 | *quanta = 4; |
1586 | break; |
1587 | case 8: |
1588 | case 9: |
1589 | *quanta = 5; |
1590 | break; |
1591 | case 10: |
1592 | case 11: |
1593 | *quanta = 6; |
1594 | break; |
1595 | case 12: |
1596 | case 13: |
1597 | *quanta = 7; |
1598 | break; |
1599 | case 14: |
1600 | case 15: /* when immediate data is present */ |
1601 | *quanta = 8; |
1602 | break; |
1603 | default: |
1604 | return -EINVAL; |
1605 | } |
1606 | |
1607 | return 0; |
1608 | } |
1609 | |
1610 | /** |
1611 | * irdma_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ |
1612 | * @frag_cnt: number of fragments |
1613 | * @wqe_size: size in bytes given frag_cnt |
1614 | */ |
1615 | int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size) |
1616 | { |
1617 | switch (frag_cnt) { |
1618 | case 0: |
1619 | case 1: |
1620 | *wqe_size = 32; |
1621 | break; |
1622 | case 2: |
1623 | case 3: |
1624 | *wqe_size = 64; |
1625 | break; |
1626 | case 4: |
1627 | case 5: |
1628 | case 6: |
1629 | case 7: |
1630 | *wqe_size = 128; |
1631 | break; |
1632 | case 8: |
1633 | case 9: |
1634 | case 10: |
1635 | case 11: |
1636 | case 12: |
1637 | case 13: |
1638 | case 14: |
1639 | *wqe_size = 256; |
1640 | break; |
1641 | default: |
1642 | return -EINVAL; |
1643 | } |
1644 | |
1645 | return 0; |
1646 | } |
1647 | |