1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2015 - 2021 Intel Corporation */ |
3 | #include <linux/etherdevice.h> |
4 | |
5 | #include "osdep.h" |
6 | #include "hmc.h" |
7 | #include "defs.h" |
8 | #include "type.h" |
9 | #include "ws.h" |
10 | #include "protos.h" |
11 | |
12 | /** |
13 | * irdma_get_qp_from_list - get next qp from a list |
14 | * @head: Listhead of qp's |
15 | * @qp: current qp |
16 | */ |
17 | struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head, |
18 | struct irdma_sc_qp *qp) |
19 | { |
20 | struct list_head *lastentry; |
21 | struct list_head *entry = NULL; |
22 | |
23 | if (list_empty(head)) |
24 | return NULL; |
25 | |
26 | if (!qp) { |
27 | entry = head->next; |
28 | } else { |
29 | lastentry = &qp->list; |
30 | entry = lastentry->next; |
31 | if (entry == head) |
32 | return NULL; |
33 | } |
34 | |
35 | return container_of(entry, struct irdma_sc_qp, list); |
36 | } |
37 | |
38 | /** |
39 | * irdma_sc_suspend_resume_qps - suspend/resume all qp's on VSI |
40 | * @vsi: the VSI struct pointer |
41 | * @op: Set to IRDMA_OP_RESUME or IRDMA_OP_SUSPEND |
42 | */ |
43 | void irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 op) |
44 | { |
45 | struct irdma_sc_qp *qp = NULL; |
46 | u8 i; |
47 | |
48 | for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { |
49 | mutex_lock(&vsi->qos[i].qos_mutex); |
50 | qp = irdma_get_qp_from_list(head: &vsi->qos[i].qplist, qp); |
51 | while (qp) { |
52 | if (op == IRDMA_OP_RESUME) { |
53 | if (!qp->dev->ws_add(vsi, i)) { |
54 | qp->qs_handle = |
55 | vsi->qos[qp->user_pri].qs_handle; |
56 | irdma_cqp_qp_suspend_resume(qp, cmd: op); |
57 | } else { |
58 | irdma_cqp_qp_suspend_resume(qp, cmd: op); |
59 | irdma_modify_qp_to_err(sc_qp: qp); |
60 | } |
61 | } else if (op == IRDMA_OP_SUSPEND) { |
62 | /* issue cqp suspend command */ |
63 | if (!irdma_cqp_qp_suspend_resume(qp, cmd: op)) |
64 | atomic_inc(v: &vsi->qp_suspend_reqs); |
65 | } |
66 | qp = irdma_get_qp_from_list(head: &vsi->qos[i].qplist, qp); |
67 | } |
68 | mutex_unlock(lock: &vsi->qos[i].qos_mutex); |
69 | } |
70 | } |
71 | |
72 | static void irdma_set_qos_info(struct irdma_sc_vsi *vsi, |
73 | struct irdma_l2params *l2p) |
74 | { |
75 | u8 i; |
76 | |
77 | vsi->qos_rel_bw = l2p->vsi_rel_bw; |
78 | vsi->qos_prio_type = l2p->vsi_prio_type; |
79 | vsi->dscp_mode = l2p->dscp_mode; |
80 | if (l2p->dscp_mode) { |
81 | memcpy(vsi->dscp_map, l2p->dscp_map, sizeof(vsi->dscp_map)); |
82 | for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) |
83 | l2p->up2tc[i] = i; |
84 | } |
85 | for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { |
86 | if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) |
87 | vsi->qos[i].qs_handle = l2p->qs_handle_list[i]; |
88 | vsi->qos[i].traffic_class = l2p->up2tc[i]; |
89 | vsi->qos[i].rel_bw = |
90 | l2p->tc_info[vsi->qos[i].traffic_class].rel_bw; |
91 | vsi->qos[i].prio_type = |
92 | l2p->tc_info[vsi->qos[i].traffic_class].prio_type; |
93 | vsi->qos[i].valid = false; |
94 | } |
95 | } |
96 | |
97 | /** |
98 | * irdma_change_l2params - given the new l2 parameters, change all qp |
99 | * @vsi: RDMA VSI pointer |
100 | * @l2params: New parameters from l2 |
101 | */ |
102 | void irdma_change_l2params(struct irdma_sc_vsi *vsi, |
103 | struct irdma_l2params *l2params) |
104 | { |
105 | if (l2params->mtu_changed) { |
106 | vsi->mtu = l2params->mtu; |
107 | if (vsi->ieq) |
108 | irdma_reinitialize_ieq(vsi); |
109 | } |
110 | |
111 | if (!l2params->tc_changed) |
112 | return; |
113 | |
114 | vsi->tc_change_pending = false; |
115 | irdma_set_qos_info(vsi, l2p: l2params); |
116 | irdma_sc_suspend_resume_qps(vsi, op: IRDMA_OP_RESUME); |
117 | } |
118 | |
119 | /** |
120 | * irdma_qp_rem_qos - remove qp from qos lists during destroy qp |
121 | * @qp: qp to be removed from qos |
122 | */ |
123 | void irdma_qp_rem_qos(struct irdma_sc_qp *qp) |
124 | { |
125 | struct irdma_sc_vsi *vsi = qp->vsi; |
126 | |
127 | ibdev_dbg(to_ibdev(qp->dev), |
128 | "DCB: DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n" , |
129 | qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, |
130 | qp->on_qoslist); |
131 | mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); |
132 | if (qp->on_qoslist) { |
133 | qp->on_qoslist = false; |
134 | list_del(entry: &qp->list); |
135 | } |
136 | mutex_unlock(lock: &vsi->qos[qp->user_pri].qos_mutex); |
137 | } |
138 | |
139 | /** |
140 | * irdma_qp_add_qos - called during setctx for qp to be added to qos |
141 | * @qp: qp to be added to qos |
142 | */ |
143 | void irdma_qp_add_qos(struct irdma_sc_qp *qp) |
144 | { |
145 | struct irdma_sc_vsi *vsi = qp->vsi; |
146 | |
147 | ibdev_dbg(to_ibdev(qp->dev), |
148 | "DCB: DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n" , |
149 | qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, |
150 | qp->on_qoslist); |
151 | mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); |
152 | if (!qp->on_qoslist) { |
153 | list_add(new: &qp->list, head: &vsi->qos[qp->user_pri].qplist); |
154 | qp->on_qoslist = true; |
155 | qp->qs_handle = vsi->qos[qp->user_pri].qs_handle; |
156 | } |
157 | mutex_unlock(lock: &vsi->qos[qp->user_pri].qos_mutex); |
158 | } |
159 | |
160 | /** |
161 | * irdma_sc_pd_init - initialize sc pd struct |
162 | * @dev: sc device struct |
163 | * @pd: sc pd ptr |
164 | * @pd_id: pd_id for allocated pd |
165 | * @abi_ver: User/Kernel ABI version |
166 | */ |
167 | void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id, |
168 | int abi_ver) |
169 | { |
170 | pd->pd_id = pd_id; |
171 | pd->abi_ver = abi_ver; |
172 | pd->dev = dev; |
173 | } |
174 | |
175 | /** |
176 | * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry |
177 | * @cqp: struct for cqp hw |
178 | * @info: arp entry information |
179 | * @scratch: u64 saved to be used during cqp completion |
180 | * @post_sq: flag for cqp db to ring |
181 | */ |
182 | static int irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp, |
183 | struct irdma_add_arp_cache_entry_info *info, |
184 | u64 scratch, bool post_sq) |
185 | { |
186 | __le64 *wqe; |
187 | u64 hdr; |
188 | |
189 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
190 | if (!wqe) |
191 | return -ENOMEM; |
192 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: info->reach_max); |
193 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: ether_addr_to_u64(addr: info->mac_addr)); |
194 | |
195 | hdr = info->arp_index | |
196 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) | |
197 | FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, (info->permanent ? 1 : 0)) | |
198 | FIELD_PREP(IRDMA_CQPSQ_MAT_ENTRYVALID, 1) | |
199 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
200 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
201 | |
202 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
203 | |
204 | print_hex_dump_debug("WQE: ARP_CACHE_ENTRY WQE" , DUMP_PREFIX_OFFSET, |
205 | 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
206 | if (post_sq) |
207 | irdma_sc_cqp_post_sq(cqp); |
208 | |
209 | return 0; |
210 | } |
211 | |
212 | /** |
213 | * irdma_sc_del_arp_cache_entry - dele arp cache entry |
214 | * @cqp: struct for cqp hw |
215 | * @scratch: u64 saved to be used during cqp completion |
216 | * @arp_index: arp index to delete arp entry |
217 | * @post_sq: flag for cqp db to ring |
218 | */ |
219 | static int irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch, |
220 | u16 arp_index, bool post_sq) |
221 | { |
222 | __le64 *wqe; |
223 | u64 hdr; |
224 | |
225 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
226 | if (!wqe) |
227 | return -ENOMEM; |
228 | |
229 | hdr = arp_index | |
230 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) | |
231 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
232 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
233 | |
234 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
235 | |
236 | print_hex_dump_debug("WQE: ARP_CACHE_DEL_ENTRY WQE" , |
237 | DUMP_PREFIX_OFFSET, 16, 8, wqe, |
238 | IRDMA_CQP_WQE_SIZE * 8, false); |
239 | if (post_sq) |
240 | irdma_sc_cqp_post_sq(cqp); |
241 | |
242 | return 0; |
243 | } |
244 | |
245 | /** |
246 | * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries |
247 | * @cqp: struct for cqp hw |
248 | * @info: info for apbvt entry to add or delete |
249 | * @scratch: u64 saved to be used during cqp completion |
250 | * @post_sq: flag for cqp db to ring |
251 | */ |
252 | static int irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp, |
253 | struct irdma_apbvt_info *info, |
254 | u64 scratch, bool post_sq) |
255 | { |
256 | __le64 *wqe; |
257 | u64 hdr; |
258 | |
259 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
260 | if (!wqe) |
261 | return -ENOMEM; |
262 | |
263 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: info->port); |
264 | |
265 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_APBVT) | |
266 | FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info->add) | |
267 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
268 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
269 | |
270 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
271 | |
272 | print_hex_dump_debug("WQE: MANAGE_APBVT WQE" , DUMP_PREFIX_OFFSET, 16, |
273 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
274 | if (post_sq) |
275 | irdma_sc_cqp_post_sq(cqp); |
276 | |
277 | return 0; |
278 | } |
279 | |
280 | /** |
281 | * irdma_sc_manage_qhash_table_entry - manage quad hash entries |
282 | * @cqp: struct for cqp hw |
283 | * @info: info for quad hash to manage |
284 | * @scratch: u64 saved to be used during cqp completion |
285 | * @post_sq: flag for cqp db to ring |
286 | * |
287 | * This is called before connection establishment is started. |
288 | * For passive connections, when listener is created, it will |
289 | * call with entry type of IRDMA_QHASH_TYPE_TCP_SYN with local |
290 | * ip address and tcp port. When SYN is received (passive |
291 | * connections) or sent (active connections), this routine is |
292 | * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED |
293 | * and quad is passed in info. |
294 | * |
295 | * When iwarp connection is done and its state moves to RTS, the |
296 | * quad hash entry in the hardware will point to iwarp's qp |
297 | * number and requires no calls from the driver. |
298 | */ |
299 | static int |
300 | irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp, |
301 | struct irdma_qhash_table_info *info, |
302 | u64 scratch, bool post_sq) |
303 | { |
304 | __le64 *wqe; |
305 | u64 qw1 = 0; |
306 | u64 qw2 = 0; |
307 | u64 temp; |
308 | struct irdma_sc_vsi *vsi = info->vsi; |
309 | |
310 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
311 | if (!wqe) |
312 | return -ENOMEM; |
313 | |
314 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: ether_addr_to_u64(addr: info->mac_addr)); |
315 | |
316 | qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) | |
317 | FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port); |
318 | if (info->ipv4_valid) { |
319 | set_64bit_val(wqe_words: wqe, byte_index: 48, |
320 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[0])); |
321 | } else { |
322 | set_64bit_val(wqe_words: wqe, byte_index: 56, |
323 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->dest_ip[0]) | |
324 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->dest_ip[1])); |
325 | |
326 | set_64bit_val(wqe_words: wqe, byte_index: 48, |
327 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->dest_ip[2]) | |
328 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[3])); |
329 | } |
330 | qw2 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QS_HANDLE, |
331 | vsi->qos[info->user_pri].qs_handle); |
332 | if (info->vlan_valid) |
333 | qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info->vlan_id); |
334 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: qw2); |
335 | if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) { |
336 | qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info->src_port); |
337 | if (!info->ipv4_valid) { |
338 | set_64bit_val(wqe_words: wqe, byte_index: 40, |
339 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->src_ip[0]) | |
340 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->src_ip[1])); |
341 | set_64bit_val(wqe_words: wqe, byte_index: 32, |
342 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->src_ip[2]) | |
343 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[3])); |
344 | } else { |
345 | set_64bit_val(wqe_words: wqe, byte_index: 32, |
346 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[0])); |
347 | } |
348 | } |
349 | |
350 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: qw1); |
351 | temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) | |
352 | FIELD_PREP(IRDMA_CQPSQ_QHASH_OPCODE, |
353 | IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY) | |
354 | FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info->manage) | |
355 | FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info->ipv4_valid) | |
356 | FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info->vlan_valid) | |
357 | FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info->entry_type); |
358 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
359 | |
360 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: temp); |
361 | |
362 | print_hex_dump_debug("WQE: MANAGE_QHASH WQE" , DUMP_PREFIX_OFFSET, 16, |
363 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
364 | if (post_sq) |
365 | irdma_sc_cqp_post_sq(cqp); |
366 | |
367 | return 0; |
368 | } |
369 | |
370 | /** |
371 | * irdma_sc_qp_init - initialize qp |
372 | * @qp: sc qp |
373 | * @info: initialization qp info |
374 | */ |
375 | int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info) |
376 | { |
377 | int ret_code; |
378 | u32 pble_obj_cnt; |
379 | u16 wqe_size; |
380 | |
381 | if (info->qp_uk_init_info.max_sq_frag_cnt > |
382 | info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags || |
383 | info->qp_uk_init_info.max_rq_frag_cnt > |
384 | info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags) |
385 | return -EINVAL; |
386 | |
387 | qp->dev = info->pd->dev; |
388 | qp->vsi = info->vsi; |
389 | qp->ieq_qp = info->vsi->exception_lan_q; |
390 | qp->sq_pa = info->sq_pa; |
391 | qp->rq_pa = info->rq_pa; |
392 | qp->hw_host_ctx_pa = info->host_ctx_pa; |
393 | qp->q2_pa = info->q2_pa; |
394 | qp->shadow_area_pa = info->shadow_area_pa; |
395 | qp->q2_buf = info->q2; |
396 | qp->pd = info->pd; |
397 | qp->hw_host_ctx = info->host_ctx; |
398 | info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db; |
399 | ret_code = irdma_uk_qp_init(qp: &qp->qp_uk, info: &info->qp_uk_init_info); |
400 | if (ret_code) |
401 | return ret_code; |
402 | |
403 | qp->virtual_map = info->virtual_map; |
404 | pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; |
405 | |
406 | if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) || |
407 | (info->virtual_map && info->rq_pa >= pble_obj_cnt)) |
408 | return -EINVAL; |
409 | |
410 | qp->llp_stream_handle = (void *)(-1); |
411 | qp->hw_sq_size = irdma_get_encoded_wqe_size(wqsize: qp->qp_uk.sq_ring.size, |
412 | queue_type: IRDMA_QUEUE_TYPE_SQ_RQ); |
413 | ibdev_dbg(to_ibdev(qp->dev), |
414 | "WQE: hw_sq_size[%04d] sq_ring.size[%04d]\n" , |
415 | qp->hw_sq_size, qp->qp_uk.sq_ring.size); |
416 | if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1 && qp->pd->abi_ver > 4) |
417 | wqe_size = IRDMA_WQE_SIZE_128; |
418 | else |
419 | ret_code = irdma_fragcnt_to_wqesize_rq(frag_cnt: qp->qp_uk.max_rq_frag_cnt, |
420 | wqe_size: &wqe_size); |
421 | if (ret_code) |
422 | return ret_code; |
423 | |
424 | qp->hw_rq_size = irdma_get_encoded_wqe_size(wqsize: qp->qp_uk.rq_size * |
425 | (wqe_size / IRDMA_QP_WQE_MIN_SIZE), queue_type: IRDMA_QUEUE_TYPE_SQ_RQ); |
426 | ibdev_dbg(to_ibdev(qp->dev), |
427 | "WQE: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n" , |
428 | qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size); |
429 | qp->sq_tph_val = info->sq_tph_val; |
430 | qp->rq_tph_val = info->rq_tph_val; |
431 | qp->sq_tph_en = info->sq_tph_en; |
432 | qp->rq_tph_en = info->rq_tph_en; |
433 | qp->rcv_tph_en = info->rcv_tph_en; |
434 | qp->xmit_tph_en = info->xmit_tph_en; |
435 | qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq; |
436 | qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle; |
437 | |
438 | return 0; |
439 | } |
440 | |
441 | /** |
442 | * irdma_sc_qp_create - create qp |
443 | * @qp: sc qp |
444 | * @info: qp create info |
445 | * @scratch: u64 saved to be used during cqp completion |
446 | * @post_sq: flag for cqp db to ring |
447 | */ |
448 | int irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_create_qp_info *info, |
449 | u64 scratch, bool post_sq) |
450 | { |
451 | struct irdma_sc_cqp *cqp; |
452 | __le64 *wqe; |
453 | u64 hdr; |
454 | |
455 | cqp = qp->dev->cqp; |
456 | if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id || |
457 | qp->qp_uk.qp_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt) |
458 | return -EINVAL; |
459 | |
460 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
461 | if (!wqe) |
462 | return -ENOMEM; |
463 | |
464 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: qp->hw_host_ctx_pa); |
465 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: qp->shadow_area_pa); |
466 | |
467 | hdr = qp->qp_uk.qp_id | |
468 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) | |
469 | FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, (info->ord_valid ? 1 : 0)) | |
470 | FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) | |
471 | FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) | |
472 | FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | |
473 | FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) | |
474 | FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) | |
475 | FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) | |
476 | FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID, |
477 | info->arp_cache_idx_valid) | |
478 | FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) | |
479 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
480 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
481 | |
482 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
483 | |
484 | print_hex_dump_debug("WQE: QP_CREATE WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
485 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
486 | if (post_sq) |
487 | irdma_sc_cqp_post_sq(cqp); |
488 | |
489 | return 0; |
490 | } |
491 | |
492 | /** |
493 | * irdma_sc_qp_modify - modify qp cqp wqe |
494 | * @qp: sc qp |
495 | * @info: modify qp info |
496 | * @scratch: u64 saved to be used during cqp completion |
497 | * @post_sq: flag for cqp db to ring |
498 | */ |
499 | int irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info, |
500 | u64 scratch, bool post_sq) |
501 | { |
502 | __le64 *wqe; |
503 | struct irdma_sc_cqp *cqp; |
504 | u64 hdr; |
505 | u8 term_actions = 0; |
506 | u8 term_len = 0; |
507 | |
508 | cqp = qp->dev->cqp; |
509 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
510 | if (!wqe) |
511 | return -ENOMEM; |
512 | |
513 | if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) { |
514 | if (info->dont_send_fin) |
515 | term_actions += IRDMAQP_TERM_SEND_TERM_ONLY; |
516 | if (info->dont_send_term) |
517 | term_actions += IRDMAQP_TERM_SEND_FIN_ONLY; |
518 | if (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN || |
519 | term_actions == IRDMAQP_TERM_SEND_TERM_ONLY) |
520 | term_len = info->termlen; |
521 | } |
522 | |
523 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
524 | FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info->new_mss) | |
525 | FIELD_PREP(IRDMA_CQPSQ_QP_TERMLEN, term_len)); |
526 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: qp->hw_host_ctx_pa); |
527 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: qp->shadow_area_pa); |
528 | |
529 | hdr = qp->qp_uk.qp_id | |
530 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_QP) | |
531 | FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) | |
532 | FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) | |
533 | FIELD_PREP(IRDMA_CQPSQ_QP_CACHEDVARVALID, |
534 | info->cached_var_valid) | |
535 | FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) | |
536 | FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) | |
537 | FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) | |
538 | FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) | |
539 | FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | |
540 | FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info->mss_change) | |
541 | FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, |
542 | info->remove_hash_idx) | |
543 | FIELD_PREP(IRDMA_CQPSQ_QP_TERMACT, term_actions) | |
544 | FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info->reset_tcp_conn) | |
545 | FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID, |
546 | info->arp_cache_idx_valid) | |
547 | FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) | |
548 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
549 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
550 | |
551 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
552 | |
553 | print_hex_dump_debug("WQE: QP_MODIFY WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
554 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
555 | if (post_sq) |
556 | irdma_sc_cqp_post_sq(cqp); |
557 | |
558 | return 0; |
559 | } |
560 | |
561 | /** |
562 | * irdma_sc_qp_destroy - cqp destroy qp |
563 | * @qp: sc qp |
564 | * @scratch: u64 saved to be used during cqp completion |
565 | * @remove_hash_idx: flag if to remove hash idx |
566 | * @ignore_mw_bnd: memory window bind flag |
567 | * @post_sq: flag for cqp db to ring |
568 | */ |
569 | int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch, |
570 | bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq) |
571 | { |
572 | __le64 *wqe; |
573 | struct irdma_sc_cqp *cqp; |
574 | u64 hdr; |
575 | |
576 | cqp = qp->dev->cqp; |
577 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
578 | if (!wqe) |
579 | return -ENOMEM; |
580 | |
581 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: qp->hw_host_ctx_pa); |
582 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: qp->shadow_area_pa); |
583 | |
584 | hdr = qp->qp_uk.qp_id | |
585 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_QP) | |
586 | FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | |
587 | FIELD_PREP(IRDMA_CQPSQ_QP_IGNOREMWBOUND, ignore_mw_bnd) | |
588 | FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, remove_hash_idx) | |
589 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
590 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
591 | |
592 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
593 | |
594 | print_hex_dump_debug("WQE: QP_DESTROY WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
595 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
596 | if (post_sq) |
597 | irdma_sc_cqp_post_sq(cqp); |
598 | |
599 | return 0; |
600 | } |
601 | |
602 | /** |
603 | * irdma_sc_get_encoded_ird_size - |
604 | * @ird_size: IRD size |
605 | * The ird from the connection is rounded to a supported HW setting and then encoded |
606 | * for ird_size field of qp_ctx. Consumers are expected to provide valid ird size based |
607 | * on hardware attributes. IRD size defaults to a value of 4 in case of invalid input |
608 | */ |
609 | static u8 irdma_sc_get_encoded_ird_size(u16 ird_size) |
610 | { |
611 | switch (ird_size ? |
612 | roundup_pow_of_two(2 * ird_size) : 4) { |
613 | case 256: |
614 | return IRDMA_IRD_HW_SIZE_256; |
615 | case 128: |
616 | return IRDMA_IRD_HW_SIZE_128; |
617 | case 64: |
618 | case 32: |
619 | return IRDMA_IRD_HW_SIZE_64; |
620 | case 16: |
621 | case 8: |
622 | return IRDMA_IRD_HW_SIZE_16; |
623 | case 4: |
624 | default: |
625 | break; |
626 | } |
627 | |
628 | return IRDMA_IRD_HW_SIZE_4; |
629 | } |
630 | |
631 | /** |
632 | * irdma_sc_qp_setctx_roce - set qp's context |
633 | * @qp: sc qp |
634 | * @qp_ctx: context ptr |
635 | * @info: ctx info |
636 | */ |
637 | void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx, |
638 | struct irdma_qp_host_ctx_info *info) |
639 | { |
640 | struct irdma_roce_offload_info *roce_info; |
641 | struct irdma_udp_offload_info *udp; |
642 | u8 push_mode_en; |
643 | u32 push_idx; |
644 | |
645 | roce_info = info->roce_info; |
646 | udp = info->udp_info; |
647 | qp->user_pri = info->user_pri; |
648 | if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) { |
649 | push_mode_en = 0; |
650 | push_idx = 0; |
651 | } else { |
652 | push_mode_en = 1; |
653 | push_idx = qp->push_idx; |
654 | } |
655 | set_64bit_val(wqe_words: qp_ctx, byte_index: 0, |
656 | FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) | |
657 | FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) | |
658 | FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) | |
659 | FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) | |
660 | FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) | |
661 | FIELD_PREP(IRDMAQPC_PPIDX, push_idx) | |
662 | FIELD_PREP(IRDMAQPC_PMENA, push_mode_en) | |
663 | FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info->pd_id >> 16) | |
664 | FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info->dctcp_en) | |
665 | FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info->err_rq_idx_valid) | |
666 | FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) | |
667 | FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) | |
668 | FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) | |
669 | FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag)); |
670 | set_64bit_val(wqe_words: qp_ctx, byte_index: 8, val: qp->sq_pa); |
671 | set_64bit_val(wqe_words: qp_ctx, byte_index: 16, val: qp->rq_pa); |
672 | if ((roce_info->dcqcn_en || roce_info->dctcp_en) && |
673 | !(udp->tos & 0x03)) |
674 | udp->tos |= ECN_CODE_PT_VAL; |
675 | set_64bit_val(wqe_words: qp_ctx, byte_index: 24, |
676 | FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) | |
677 | FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size) | |
678 | FIELD_PREP(IRDMAQPC_TTL, udp->ttl) | FIELD_PREP(IRDMAQPC_TOS, udp->tos) | |
679 | FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp->src_port) | |
680 | FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp->dst_port)); |
681 | set_64bit_val(wqe_words: qp_ctx, byte_index: 32, |
682 | FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp->dest_ip_addr[2]) | |
683 | FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp->dest_ip_addr[3])); |
684 | set_64bit_val(wqe_words: qp_ctx, byte_index: 40, |
685 | FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp->dest_ip_addr[0]) | |
686 | FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp->dest_ip_addr[1])); |
687 | set_64bit_val(wqe_words: qp_ctx, byte_index: 48, |
688 | FIELD_PREP(IRDMAQPC_SNDMSS, udp->snd_mss) | |
689 | FIELD_PREP(IRDMAQPC_VLANTAG, udp->vlan_tag) | |
690 | FIELD_PREP(IRDMAQPC_ARPIDX, udp->arp_idx)); |
691 | set_64bit_val(wqe_words: qp_ctx, byte_index: 56, |
692 | FIELD_PREP(IRDMAQPC_PKEY, roce_info->p_key) | |
693 | FIELD_PREP(IRDMAQPC_PDIDX, roce_info->pd_id) | |
694 | FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info->ack_credits) | |
695 | FIELD_PREP(IRDMAQPC_FLOWLABEL, udp->flow_label)); |
696 | set_64bit_val(wqe_words: qp_ctx, byte_index: 64, |
697 | FIELD_PREP(IRDMAQPC_QKEY, roce_info->qkey) | |
698 | FIELD_PREP(IRDMAQPC_DESTQP, roce_info->dest_qp)); |
699 | set_64bit_val(wqe_words: qp_ctx, byte_index: 80, |
700 | FIELD_PREP(IRDMAQPC_PSNNXT, udp->psn_nxt) | |
701 | FIELD_PREP(IRDMAQPC_LSN, udp->lsn)); |
702 | set_64bit_val(wqe_words: qp_ctx, byte_index: 88, |
703 | FIELD_PREP(IRDMAQPC_EPSN, udp->epsn)); |
704 | set_64bit_val(wqe_words: qp_ctx, byte_index: 96, |
705 | FIELD_PREP(IRDMAQPC_PSNMAX, udp->psn_max) | |
706 | FIELD_PREP(IRDMAQPC_PSNUNA, udp->psn_una)); |
707 | set_64bit_val(wqe_words: qp_ctx, byte_index: 112, |
708 | FIELD_PREP(IRDMAQPC_CWNDROCE, udp->cwnd)); |
709 | set_64bit_val(wqe_words: qp_ctx, byte_index: 128, |
710 | FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info->err_rq_idx) | |
711 | FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp->rnr_nak_thresh) | |
712 | FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp->rexmit_thresh) | |
713 | FIELD_PREP(IRDMAQPC_RTOMIN, roce_info->rtomin)); |
714 | set_64bit_val(wqe_words: qp_ctx, byte_index: 136, |
715 | FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) | |
716 | FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num)); |
717 | set_64bit_val(wqe_words: qp_ctx, byte_index: 144, |
718 | FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); |
719 | set_64bit_val(wqe_words: qp_ctx, byte_index: 152, val: ether_addr_to_u64(addr: roce_info->mac_addr) << 16); |
720 | set_64bit_val(wqe_words: qp_ctx, byte_index: 160, |
721 | FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) | |
722 | FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) | |
723 | FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) | |
724 | FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) | |
725 | FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | |
726 | FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) | |
727 | FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) | |
728 | FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) | |
729 | FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) | |
730 | FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info->fw_cc_enable) | |
731 | FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info->udprivcq_en) | |
732 | FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) | |
733 | FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en)); |
734 | set_64bit_val(wqe_words: qp_ctx, byte_index: 168, |
735 | FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx)); |
736 | set_64bit_val(wqe_words: qp_ctx, byte_index: 176, |
737 | FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) | |
738 | FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) | |
739 | FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle)); |
740 | set_64bit_val(wqe_words: qp_ctx, byte_index: 184, |
741 | FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp->local_ipaddr[3]) | |
742 | FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp->local_ipaddr[2])); |
743 | set_64bit_val(wqe_words: qp_ctx, byte_index: 192, |
744 | FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp->local_ipaddr[1]) | |
745 | FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0])); |
746 | set_64bit_val(wqe_words: qp_ctx, byte_index: 200, |
747 | FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) | |
748 | FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low)); |
749 | set_64bit_val(wqe_words: qp_ctx, byte_index: 208, |
750 | FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx)); |
751 | |
752 | print_hex_dump_debug("WQE: QP_HOST CTX WQE" , DUMP_PREFIX_OFFSET, 16, |
753 | 8, qp_ctx, IRDMA_QP_CTX_SIZE, false); |
754 | } |
755 | |
756 | /* irdma_sc_alloc_local_mac_entry - allocate a mac entry |
757 | * @cqp: struct for cqp hw |
758 | * @scratch: u64 saved to be used during cqp completion |
759 | * @post_sq: flag for cqp db to ring |
760 | */ |
761 | static int irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, |
762 | bool post_sq) |
763 | { |
764 | __le64 *wqe; |
765 | u64 hdr; |
766 | |
767 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
768 | if (!wqe) |
769 | return -ENOMEM; |
770 | |
771 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, |
772 | IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY) | |
773 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
774 | |
775 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
776 | |
777 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
778 | |
779 | print_hex_dump_debug("WQE: ALLOCATE_LOCAL_MAC WQE" , |
780 | DUMP_PREFIX_OFFSET, 16, 8, wqe, |
781 | IRDMA_CQP_WQE_SIZE * 8, false); |
782 | |
783 | if (post_sq) |
784 | irdma_sc_cqp_post_sq(cqp); |
785 | return 0; |
786 | } |
787 | |
788 | /** |
789 | * irdma_sc_add_local_mac_entry - add mac enry |
790 | * @cqp: struct for cqp hw |
791 | * @info:mac addr info |
792 | * @scratch: u64 saved to be used during cqp completion |
793 | * @post_sq: flag for cqp db to ring |
794 | */ |
795 | static int irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp, |
796 | struct irdma_local_mac_entry_info *info, |
797 | u64 scratch, bool post_sq) |
798 | { |
799 | __le64 *wqe; |
800 | u64 ; |
801 | |
802 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
803 | if (!wqe) |
804 | return -ENOMEM; |
805 | |
806 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: ether_addr_to_u64(addr: info->mac_addr)); |
807 | |
808 | header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) | |
809 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, |
810 | IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) | |
811 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
812 | |
813 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
814 | |
815 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: header); |
816 | |
817 | print_hex_dump_debug("WQE: ADD_LOCAL_MAC WQE" , DUMP_PREFIX_OFFSET, 16, |
818 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
819 | |
820 | if (post_sq) |
821 | irdma_sc_cqp_post_sq(cqp); |
822 | return 0; |
823 | } |
824 | |
825 | /** |
826 | * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac |
827 | * @cqp: struct for cqp hw |
828 | * @scratch: u64 saved to be used during cqp completion |
829 | * @entry_idx: index of mac entry |
830 | * @ignore_ref_count: to force mac adde delete |
831 | * @post_sq: flag for cqp db to ring |
832 | */ |
833 | static int irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, |
834 | u16 entry_idx, u8 ignore_ref_count, |
835 | bool post_sq) |
836 | { |
837 | __le64 *wqe; |
838 | u64 ; |
839 | |
840 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
841 | if (!wqe) |
842 | return -ENOMEM; |
843 | header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, entry_idx) | |
844 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, |
845 | IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) | |
846 | FIELD_PREP(IRDMA_CQPSQ_MLM_FREEENTRY, 1) | |
847 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | |
848 | FIELD_PREP(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT, ignore_ref_count); |
849 | |
850 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
851 | |
852 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: header); |
853 | |
854 | print_hex_dump_debug("WQE: DEL_LOCAL_MAC_IPADDR WQE" , |
855 | DUMP_PREFIX_OFFSET, 16, 8, wqe, |
856 | IRDMA_CQP_WQE_SIZE * 8, false); |
857 | |
858 | if (post_sq) |
859 | irdma_sc_cqp_post_sq(cqp); |
860 | return 0; |
861 | } |
862 | |
863 | /** |
864 | * irdma_sc_qp_setctx - set qp's context |
865 | * @qp: sc qp |
866 | * @qp_ctx: context ptr |
867 | * @info: ctx info |
868 | */ |
869 | void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx, |
870 | struct irdma_qp_host_ctx_info *info) |
871 | { |
872 | struct irdma_iwarp_offload_info *iw; |
873 | struct irdma_tcp_offload_info *tcp; |
874 | struct irdma_sc_dev *dev; |
875 | u8 push_mode_en; |
876 | u32 push_idx; |
877 | u64 qw0, qw3, qw7 = 0, qw16 = 0; |
878 | u64 mac = 0; |
879 | |
880 | iw = info->iwarp_info; |
881 | tcp = info->tcp_info; |
882 | dev = qp->dev; |
883 | if (iw->rcv_mark_en) { |
884 | qp->pfpdu.marker_len = 4; |
885 | qp->pfpdu.rcv_start_seq = tcp->rcv_nxt; |
886 | } |
887 | qp->user_pri = info->user_pri; |
888 | if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) { |
889 | push_mode_en = 0; |
890 | push_idx = 0; |
891 | } else { |
892 | push_mode_en = 1; |
893 | push_idx = qp->push_idx; |
894 | } |
895 | qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) | |
896 | FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) | |
897 | FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) | |
898 | FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) | |
899 | FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) | |
900 | FIELD_PREP(IRDMAQPC_PPIDX, push_idx) | |
901 | FIELD_PREP(IRDMAQPC_PMENA, push_mode_en); |
902 | |
903 | set_64bit_val(wqe_words: qp_ctx, byte_index: 8, val: qp->sq_pa); |
904 | set_64bit_val(wqe_words: qp_ctx, byte_index: 16, val: qp->rq_pa); |
905 | |
906 | qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) | |
907 | FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size); |
908 | if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) |
909 | qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, |
910 | qp->src_mac_addr_idx); |
911 | set_64bit_val(wqe_words: qp_ctx, byte_index: 136, |
912 | FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) | |
913 | FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num)); |
914 | set_64bit_val(wqe_words: qp_ctx, byte_index: 168, |
915 | FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx)); |
916 | set_64bit_val(wqe_words: qp_ctx, byte_index: 176, |
917 | FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) | |
918 | FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) | |
919 | FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle) | |
920 | FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp->ieq_qp)); |
921 | if (info->iwarp_info_valid) { |
922 | qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw->ddp_ver) | |
923 | FIELD_PREP(IRDMAQPC_RDMAP_VER, iw->rdmap_ver) | |
924 | FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw->dctcp_en) | |
925 | FIELD_PREP(IRDMAQPC_ECN_EN, iw->ecn_en) | |
926 | FIELD_PREP(IRDMAQPC_IBRDENABLE, iw->ib_rd_en) | |
927 | FIELD_PREP(IRDMAQPC_PDIDXHI, iw->pd_id >> 16) | |
928 | FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, |
929 | iw->err_rq_idx_valid); |
930 | qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw->pd_id); |
931 | qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw->err_rq_idx) | |
932 | FIELD_PREP(IRDMAQPC_RTOMIN, iw->rtomin); |
933 | set_64bit_val(wqe_words: qp_ctx, byte_index: 144, |
934 | FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) | |
935 | FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); |
936 | |
937 | if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) |
938 | mac = ether_addr_to_u64(addr: iw->mac_addr); |
939 | |
940 | set_64bit_val(wqe_words: qp_ctx, byte_index: 152, |
941 | val: mac << 16 | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent)); |
942 | set_64bit_val(wqe_words: qp_ctx, byte_index: 160, |
943 | FIELD_PREP(IRDMAQPC_ORDSIZE, iw->ord_size) | |
944 | FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw->ird_size)) | |
945 | FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) | |
946 | FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) | |
947 | FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) | |
948 | FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) | |
949 | FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) | |
950 | FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) | |
951 | FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | |
952 | FIELD_PREP(IRDMAQPC_IWARPMODE, 1) | |
953 | FIELD_PREP(IRDMAQPC_RCVMARKERS, iw->rcv_mark_en) | |
954 | FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw->align_hdrs) | |
955 | FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw->rcv_no_mpa_crc) | |
956 | FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw->rcv_mark_offset || !tcp ? iw->rcv_mark_offset : tcp->rcv_nxt) | |
957 | FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw->snd_mark_offset || !tcp ? iw->snd_mark_offset : tcp->snd_nxt) | |
958 | FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw->timely_en)); |
959 | } |
960 | if (info->tcp_info_valid) { |
961 | qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp->ipv4) | |
962 | FIELD_PREP(IRDMAQPC_NONAGLE, tcp->no_nagle) | |
963 | FIELD_PREP(IRDMAQPC_INSERTVLANTAG, |
964 | tcp->insert_vlan_tag) | |
965 | FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp->time_stamp) | |
966 | FIELD_PREP(IRDMAQPC_LIMIT, tcp->cwnd_inc_limit) | |
967 | FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp->drop_ooo_seg) | |
968 | FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp->dup_ack_thresh); |
969 | |
970 | if ((iw->ecn_en || iw->dctcp_en) && !(tcp->tos & 0x03)) |
971 | tcp->tos |= ECN_CODE_PT_VAL; |
972 | |
973 | qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp->ttl) | |
974 | FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp->avoid_stretch_ack) | |
975 | FIELD_PREP(IRDMAQPC_TOS, tcp->tos) | |
976 | FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp->src_port) | |
977 | FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp->dst_port); |
978 | if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { |
979 | qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp->src_mac_addr_idx); |
980 | |
981 | qp->src_mac_addr_idx = tcp->src_mac_addr_idx; |
982 | } |
983 | set_64bit_val(wqe_words: qp_ctx, byte_index: 32, |
984 | FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp->dest_ip_addr[2]) | |
985 | FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp->dest_ip_addr[3])); |
986 | set_64bit_val(wqe_words: qp_ctx, byte_index: 40, |
987 | FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp->dest_ip_addr[0]) | |
988 | FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp->dest_ip_addr[1])); |
989 | set_64bit_val(wqe_words: qp_ctx, byte_index: 48, |
990 | FIELD_PREP(IRDMAQPC_SNDMSS, tcp->snd_mss) | |
991 | FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp->syn_rst_handling) | |
992 | FIELD_PREP(IRDMAQPC_VLANTAG, tcp->vlan_tag) | |
993 | FIELD_PREP(IRDMAQPC_ARPIDX, tcp->arp_idx)); |
994 | qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp->flow_label) | |
995 | FIELD_PREP(IRDMAQPC_WSCALE, tcp->wscale) | |
996 | FIELD_PREP(IRDMAQPC_IGNORE_TCP_OPT, |
997 | tcp->ignore_tcp_opt) | |
998 | FIELD_PREP(IRDMAQPC_IGNORE_TCP_UNS_OPT, |
999 | tcp->ignore_tcp_uns_opt) | |
1000 | FIELD_PREP(IRDMAQPC_TCPSTATE, tcp->tcp_state) | |
1001 | FIELD_PREP(IRDMAQPC_RCVSCALE, tcp->rcv_wscale) | |
1002 | FIELD_PREP(IRDMAQPC_SNDSCALE, tcp->snd_wscale); |
1003 | set_64bit_val(wqe_words: qp_ctx, byte_index: 72, |
1004 | FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp->time_stamp_recent) | |
1005 | FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp->time_stamp_age)); |
1006 | set_64bit_val(wqe_words: qp_ctx, byte_index: 80, |
1007 | FIELD_PREP(IRDMAQPC_SNDNXT, tcp->snd_nxt) | |
1008 | FIELD_PREP(IRDMAQPC_SNDWND, tcp->snd_wnd)); |
1009 | set_64bit_val(wqe_words: qp_ctx, byte_index: 88, |
1010 | FIELD_PREP(IRDMAQPC_RCVNXT, tcp->rcv_nxt) | |
1011 | FIELD_PREP(IRDMAQPC_RCVWND, tcp->rcv_wnd)); |
1012 | set_64bit_val(wqe_words: qp_ctx, byte_index: 96, |
1013 | FIELD_PREP(IRDMAQPC_SNDMAX, tcp->snd_max) | |
1014 | FIELD_PREP(IRDMAQPC_SNDUNA, tcp->snd_una)); |
1015 | set_64bit_val(wqe_words: qp_ctx, byte_index: 104, |
1016 | FIELD_PREP(IRDMAQPC_SRTT, tcp->srtt) | |
1017 | FIELD_PREP(IRDMAQPC_RTTVAR, tcp->rtt_var)); |
1018 | set_64bit_val(wqe_words: qp_ctx, byte_index: 112, |
1019 | FIELD_PREP(IRDMAQPC_SSTHRESH, tcp->ss_thresh) | |
1020 | FIELD_PREP(IRDMAQPC_CWND, tcp->cwnd)); |
1021 | set_64bit_val(wqe_words: qp_ctx, byte_index: 120, |
1022 | FIELD_PREP(IRDMAQPC_SNDWL1, tcp->snd_wl1) | |
1023 | FIELD_PREP(IRDMAQPC_SNDWL2, tcp->snd_wl2)); |
1024 | qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp->max_snd_window) | |
1025 | FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp->rexmit_thresh); |
1026 | set_64bit_val(wqe_words: qp_ctx, byte_index: 184, |
1027 | FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp->local_ipaddr[3]) | |
1028 | FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp->local_ipaddr[2])); |
1029 | set_64bit_val(wqe_words: qp_ctx, byte_index: 192, |
1030 | FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp->local_ipaddr[1]) | |
1031 | FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp->local_ipaddr[0])); |
1032 | set_64bit_val(wqe_words: qp_ctx, byte_index: 200, |
1033 | FIELD_PREP(IRDMAQPC_THIGH, iw->t_high) | |
1034 | FIELD_PREP(IRDMAQPC_TLOW, iw->t_low)); |
1035 | set_64bit_val(wqe_words: qp_ctx, byte_index: 208, |
1036 | FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx)); |
1037 | } |
1038 | |
1039 | set_64bit_val(wqe_words: qp_ctx, byte_index: 0, val: qw0); |
1040 | set_64bit_val(wqe_words: qp_ctx, byte_index: 24, val: qw3); |
1041 | set_64bit_val(wqe_words: qp_ctx, byte_index: 56, val: qw7); |
1042 | set_64bit_val(wqe_words: qp_ctx, byte_index: 128, val: qw16); |
1043 | |
1044 | print_hex_dump_debug("WQE: QP_HOST CTX" , DUMP_PREFIX_OFFSET, 16, 8, |
1045 | qp_ctx, IRDMA_QP_CTX_SIZE, false); |
1046 | } |
1047 | |
1048 | /** |
1049 | * irdma_sc_alloc_stag - mr stag alloc |
1050 | * @dev: sc device struct |
1051 | * @info: stag info |
1052 | * @scratch: u64 saved to be used during cqp completion |
1053 | * @post_sq: flag for cqp db to ring |
1054 | */ |
1055 | static int irdma_sc_alloc_stag(struct irdma_sc_dev *dev, |
1056 | struct irdma_allocate_stag_info *info, |
1057 | u64 scratch, bool post_sq) |
1058 | { |
1059 | __le64 *wqe; |
1060 | struct irdma_sc_cqp *cqp; |
1061 | u64 hdr; |
1062 | enum irdma_page_size page_size; |
1063 | |
1064 | if (!info->total_len && !info->all_memory) |
1065 | return -EINVAL; |
1066 | |
1067 | if (info->page_size == 0x40000000) |
1068 | page_size = IRDMA_PAGE_SIZE_1G; |
1069 | else if (info->page_size == 0x200000) |
1070 | page_size = IRDMA_PAGE_SIZE_2M; |
1071 | else |
1072 | page_size = IRDMA_PAGE_SIZE_4K; |
1073 | |
1074 | cqp = dev->cqp; |
1075 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
1076 | if (!wqe) |
1077 | return -ENOMEM; |
1078 | |
1079 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1080 | FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) | |
1081 | FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len)); |
1082 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
1083 | FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); |
1084 | set_64bit_val(wqe_words: wqe, byte_index: 40, |
1085 | FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info->hmc_fcn_index)); |
1086 | |
1087 | if (info->chunk_size) |
1088 | set_64bit_val(wqe_words: wqe, byte_index: 48, |
1089 | FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_idx)); |
1090 | |
1091 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) | |
1092 | FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) | |
1093 | FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) | |
1094 | FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) | |
1095 | FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) | |
1096 | FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) | |
1097 | FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) | |
1098 | FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) | |
1099 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
1100 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
1101 | |
1102 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1103 | |
1104 | print_hex_dump_debug("WQE: ALLOC_STAG WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
1105 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
1106 | if (post_sq) |
1107 | irdma_sc_cqp_post_sq(cqp); |
1108 | |
1109 | return 0; |
1110 | } |
1111 | |
1112 | /** |
1113 | * irdma_sc_mr_reg_non_shared - non-shared mr registration |
1114 | * @dev: sc device struct |
1115 | * @info: mr info |
1116 | * @scratch: u64 saved to be used during cqp completion |
1117 | * @post_sq: flag for cqp db to ring |
1118 | */ |
1119 | static int irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev, |
1120 | struct irdma_reg_ns_stag_info *info, |
1121 | u64 scratch, bool post_sq) |
1122 | { |
1123 | __le64 *wqe; |
1124 | u64 fbo; |
1125 | struct irdma_sc_cqp *cqp; |
1126 | u64 hdr; |
1127 | u32 pble_obj_cnt; |
1128 | bool remote_access; |
1129 | u8 addr_type; |
1130 | enum irdma_page_size page_size; |
1131 | |
1132 | if (!info->total_len && !info->all_memory) |
1133 | return -EINVAL; |
1134 | |
1135 | if (info->page_size == 0x40000000) |
1136 | page_size = IRDMA_PAGE_SIZE_1G; |
1137 | else if (info->page_size == 0x200000) |
1138 | page_size = IRDMA_PAGE_SIZE_2M; |
1139 | else if (info->page_size == 0x1000) |
1140 | page_size = IRDMA_PAGE_SIZE_4K; |
1141 | else |
1142 | return -EINVAL; |
1143 | |
1144 | if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY | |
1145 | IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY)) |
1146 | remote_access = true; |
1147 | else |
1148 | remote_access = false; |
1149 | |
1150 | pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; |
1151 | if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt) |
1152 | return -EINVAL; |
1153 | |
1154 | cqp = dev->cqp; |
1155 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
1156 | if (!wqe) |
1157 | return -ENOMEM; |
1158 | fbo = info->va & (info->page_size - 1); |
1159 | |
1160 | set_64bit_val(wqe_words: wqe, byte_index: 0, |
1161 | val: (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ? |
1162 | info->va : fbo)); |
1163 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1164 | FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len) | |
1165 | FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); |
1166 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
1167 | FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info->stag_key) | |
1168 | FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); |
1169 | if (!info->chunk_size) { |
1170 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: info->reg_addr_pa); |
1171 | set_64bit_val(wqe_words: wqe, byte_index: 48, val: 0); |
1172 | } else { |
1173 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: 0); |
1174 | set_64bit_val(wqe_words: wqe, byte_index: 48, |
1175 | FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_index)); |
1176 | } |
1177 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: info->hmc_fcn_index); |
1178 | set_64bit_val(wqe_words: wqe, byte_index: 56, val: 0); |
1179 | |
1180 | addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0; |
1181 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_REG_MR) | |
1182 | FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) | |
1183 | FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) | |
1184 | FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) | |
1185 | FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) | |
1186 | FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, remote_access) | |
1187 | FIELD_PREP(IRDMA_CQPSQ_STAG_VABASEDTO, addr_type) | |
1188 | FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) | |
1189 | FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) | |
1190 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
1191 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
1192 | |
1193 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1194 | |
1195 | print_hex_dump_debug("WQE: MR_REG_NS WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
1196 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
1197 | if (post_sq) |
1198 | irdma_sc_cqp_post_sq(cqp); |
1199 | |
1200 | return 0; |
1201 | } |
1202 | |
1203 | /** |
1204 | * irdma_sc_dealloc_stag - deallocate stag |
1205 | * @dev: sc device struct |
1206 | * @info: dealloc stag info |
1207 | * @scratch: u64 saved to be used during cqp completion |
1208 | * @post_sq: flag for cqp db to ring |
1209 | */ |
1210 | static int irdma_sc_dealloc_stag(struct irdma_sc_dev *dev, |
1211 | struct irdma_dealloc_stag_info *info, |
1212 | u64 scratch, bool post_sq) |
1213 | { |
1214 | u64 hdr; |
1215 | __le64 *wqe; |
1216 | struct irdma_sc_cqp *cqp; |
1217 | |
1218 | cqp = dev->cqp; |
1219 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
1220 | if (!wqe) |
1221 | return -ENOMEM; |
1222 | |
1223 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1224 | FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); |
1225 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
1226 | FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); |
1227 | |
1228 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DEALLOC_STAG) | |
1229 | FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info->mr) | |
1230 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
1231 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
1232 | |
1233 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1234 | |
1235 | print_hex_dump_debug("WQE: DEALLOC_STAG WQE" , DUMP_PREFIX_OFFSET, 16, |
1236 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
1237 | if (post_sq) |
1238 | irdma_sc_cqp_post_sq(cqp); |
1239 | |
1240 | return 0; |
1241 | } |
1242 | |
1243 | /** |
1244 | * irdma_sc_mw_alloc - mw allocate |
1245 | * @dev: sc device struct |
1246 | * @info: memory window allocation information |
1247 | * @scratch: u64 saved to be used during cqp completion |
1248 | * @post_sq: flag for cqp db to ring |
1249 | */ |
1250 | static int irdma_sc_mw_alloc(struct irdma_sc_dev *dev, |
1251 | struct irdma_mw_alloc_info *info, u64 scratch, |
1252 | bool post_sq) |
1253 | { |
1254 | u64 hdr; |
1255 | struct irdma_sc_cqp *cqp; |
1256 | __le64 *wqe; |
1257 | |
1258 | cqp = dev->cqp; |
1259 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
1260 | if (!wqe) |
1261 | return -ENOMEM; |
1262 | |
1263 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1264 | FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); |
1265 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
1266 | FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->mw_stag_index)); |
1267 | |
1268 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) | |
1269 | FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info->mw_wide) | |
1270 | FIELD_PREP(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY, |
1271 | info->mw1_bind_dont_vldt_key) | |
1272 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
1273 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
1274 | |
1275 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1276 | |
1277 | print_hex_dump_debug("WQE: MW_ALLOC WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
1278 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
1279 | if (post_sq) |
1280 | irdma_sc_cqp_post_sq(cqp); |
1281 | |
1282 | return 0; |
1283 | } |
1284 | |
1285 | /** |
1286 | * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp |
1287 | * @qp: sc qp struct |
1288 | * @info: fast mr info |
1289 | * @post_sq: flag for cqp db to ring |
1290 | */ |
1291 | int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, |
1292 | struct irdma_fast_reg_stag_info *info, |
1293 | bool post_sq) |
1294 | { |
1295 | u64 temp, hdr; |
1296 | __le64 *wqe; |
1297 | u32 wqe_idx; |
1298 | enum irdma_page_size page_size; |
1299 | struct irdma_post_sq_info sq_info = {}; |
1300 | |
1301 | if (info->page_size == 0x40000000) |
1302 | page_size = IRDMA_PAGE_SIZE_1G; |
1303 | else if (info->page_size == 0x200000) |
1304 | page_size = IRDMA_PAGE_SIZE_2M; |
1305 | else |
1306 | page_size = IRDMA_PAGE_SIZE_4K; |
1307 | |
1308 | sq_info.wr_id = info->wr_id; |
1309 | sq_info.signaled = info->signaled; |
1310 | |
1311 | wqe = irdma_qp_get_next_send_wqe(qp: &qp->qp_uk, wqe_idx: &wqe_idx, |
1312 | IRDMA_QP_WQE_MIN_QUANTA, total_size: 0, info: &sq_info); |
1313 | if (!wqe) |
1314 | return -ENOMEM; |
1315 | |
1316 | irdma_clr_wqes(qp: &qp->qp_uk, qp_wqe_idx: wqe_idx); |
1317 | |
1318 | ibdev_dbg(to_ibdev(qp->dev), |
1319 | "MR: wr_id[%llxh] wqe_idx[%04d] location[%p]\n" , |
1320 | info->wr_id, wqe_idx, |
1321 | &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); |
1322 | |
1323 | temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? |
1324 | (uintptr_t)info->va : info->fbo; |
1325 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: temp); |
1326 | |
1327 | temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI, |
1328 | info->first_pm_pbl_index >> 16); |
1329 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1330 | FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) | |
1331 | FIELD_PREP(IRDMAQPSQ_PBLADDR >> IRDMA_HW_PAGE_SHIFT, info->reg_addr_pa)); |
1332 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
1333 | val: info->total_len | |
1334 | FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index)); |
1335 | |
1336 | hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info->stag_key) | |
1337 | FIELD_PREP(IRDMAQPSQ_STAGINDEX, info->stag_idx) | |
1338 | FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_FAST_REGISTER) | |
1339 | FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) | |
1340 | FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) | |
1341 | FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) | |
1342 | FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) | |
1343 | FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | |
1344 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | |
1345 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | |
1346 | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); |
1347 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
1348 | |
1349 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1350 | |
1351 | print_hex_dump_debug("WQE: FAST_REG WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
1352 | wqe, IRDMA_QP_WQE_MIN_SIZE, false); |
1353 | |
1354 | if (post_sq) |
1355 | irdma_uk_qp_post_wr(qp: &qp->qp_uk); |
1356 | |
1357 | return 0; |
1358 | } |
1359 | |
1360 | /** |
1361 | * irdma_sc_gen_rts_ae - request AE generated after RTS |
1362 | * @qp: sc qp struct |
1363 | */ |
1364 | static void irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp) |
1365 | { |
1366 | __le64 *wqe; |
1367 | u64 hdr; |
1368 | struct irdma_qp_uk *qp_uk; |
1369 | |
1370 | qp_uk = &qp->qp_uk; |
1371 | |
1372 | wqe = qp_uk->sq_base[1].elem; |
1373 | |
1374 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | |
1375 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) | |
1376 | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); |
1377 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
1378 | |
1379 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1380 | print_hex_dump_debug("QP: NOP W/LOCAL FENCE WQE" , DUMP_PREFIX_OFFSET, |
1381 | 16, 8, wqe, IRDMA_QP_WQE_MIN_SIZE, false); |
1382 | |
1383 | wqe = qp_uk->sq_base[2].elem; |
1384 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_GEN_RTS_AE) | |
1385 | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); |
1386 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
1387 | |
1388 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1389 | print_hex_dump_debug("QP: CONN EST WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
1390 | wqe, IRDMA_QP_WQE_MIN_SIZE, false); |
1391 | } |
1392 | |
1393 | /** |
1394 | * irdma_sc_send_lsmm - send last streaming mode message |
1395 | * @qp: sc qp struct |
1396 | * @lsmm_buf: buffer with lsmm message |
1397 | * @size: size of lsmm buffer |
1398 | * @stag: stag of lsmm buffer |
1399 | */ |
1400 | void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size, |
1401 | irdma_stag stag) |
1402 | { |
1403 | __le64 *wqe; |
1404 | u64 hdr; |
1405 | struct irdma_qp_uk *qp_uk; |
1406 | |
1407 | qp_uk = &qp->qp_uk; |
1408 | wqe = qp_uk->sq_base->elem; |
1409 | |
1410 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: (uintptr_t)lsmm_buf); |
1411 | if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { |
1412 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1413 | FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size) | |
1414 | FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, stag)); |
1415 | } else { |
1416 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1417 | FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) | |
1418 | FIELD_PREP(IRDMAQPSQ_FRAG_STAG, stag) | |
1419 | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); |
1420 | } |
1421 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: 0); |
1422 | |
1423 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) | |
1424 | FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) | |
1425 | FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) | |
1426 | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); |
1427 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
1428 | |
1429 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1430 | |
1431 | print_hex_dump_debug("WQE: SEND_LSMM WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
1432 | wqe, IRDMA_QP_WQE_MIN_SIZE, false); |
1433 | |
1434 | if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) |
1435 | irdma_sc_gen_rts_ae(qp); |
1436 | } |
1437 | |
1438 | /** |
1439 | * irdma_sc_send_rtt - send last read0 or write0 |
1440 | * @qp: sc qp struct |
1441 | * @read: Do read0 or write0 |
1442 | */ |
1443 | void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read) |
1444 | { |
1445 | __le64 *wqe; |
1446 | u64 hdr; |
1447 | struct irdma_qp_uk *qp_uk; |
1448 | |
1449 | qp_uk = &qp->qp_uk; |
1450 | wqe = qp_uk->sq_base->elem; |
1451 | |
1452 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: 0); |
1453 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: 0); |
1454 | if (read) { |
1455 | if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { |
1456 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1457 | FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, 0xabcd)); |
1458 | } else { |
1459 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1460 | val: (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); |
1461 | } |
1462 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, 0x1234) | |
1463 | FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_READ) | |
1464 | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); |
1465 | |
1466 | } else { |
1467 | if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { |
1468 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: 0); |
1469 | } else { |
1470 | set_64bit_val(wqe_words: wqe, byte_index: 8, |
1471 | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); |
1472 | } |
1473 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_WRITE) | |
1474 | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); |
1475 | } |
1476 | |
1477 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
1478 | |
1479 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
1480 | |
1481 | print_hex_dump_debug("WQE: RTR WQE" , DUMP_PREFIX_OFFSET, 16, 8, wqe, |
1482 | IRDMA_QP_WQE_MIN_SIZE, false); |
1483 | |
1484 | if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) |
1485 | irdma_sc_gen_rts_ae(qp); |
1486 | } |
1487 | |
1488 | /** |
1489 | * irdma_iwarp_opcode - determine if incoming is rdma layer |
1490 | * @info: aeq info for the packet |
1491 | * @pkt: packet for error |
1492 | */ |
1493 | static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt) |
1494 | { |
1495 | __be16 *mpa; |
1496 | u32 opcode = 0xffffffff; |
1497 | |
1498 | if (info->q2_data_written) { |
1499 | mpa = (__be16 *)pkt; |
1500 | opcode = ntohs(mpa[1]) & 0xf; |
1501 | } |
1502 | |
1503 | return opcode; |
1504 | } |
1505 | |
1506 | /** |
1507 | * irdma_locate_mpa - return pointer to mpa in the pkt |
1508 | * @pkt: packet with data |
1509 | */ |
1510 | static u8 *irdma_locate_mpa(u8 *pkt) |
1511 | { |
1512 | /* skip over ethernet header */ |
1513 | pkt += IRDMA_MAC_HLEN; |
1514 | |
1515 | /* Skip over IP and TCP headers */ |
1516 | pkt += 4 * (pkt[0] & 0x0f); |
1517 | pkt += 4 * ((pkt[12] >> 4) & 0x0f); |
1518 | |
1519 | return pkt; |
1520 | } |
1521 | |
1522 | /** |
1523 | * irdma_bld_termhdr_ctrl - setup terminate hdr control fields |
1524 | * @qp: sc qp ptr for pkt |
1525 | * @hdr: term hdr |
1526 | * @opcode: flush opcode for termhdr |
1527 | * @layer_etype: error layer + error type |
1528 | * @err: error cod ein the header |
1529 | */ |
1530 | static void irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp, |
1531 | struct irdma_terminate_hdr *hdr, |
1532 | enum irdma_flush_opcode opcode, |
1533 | u8 layer_etype, u8 err) |
1534 | { |
1535 | qp->flush_code = opcode; |
1536 | hdr->layer_etype = layer_etype; |
1537 | hdr->error_code = err; |
1538 | } |
1539 | |
1540 | /** |
1541 | * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr |
1542 | * @pkt: ptr to mpa in offending pkt |
1543 | * @hdr: term hdr |
1544 | * @copy_len: offending pkt length to be copied to term hdr |
1545 | * @is_tagged: DDP tagged or untagged |
1546 | */ |
1547 | static void irdma_bld_termhdr_ddp_rdma(u8 *pkt, struct irdma_terminate_hdr *hdr, |
1548 | int *copy_len, u8 *is_tagged) |
1549 | { |
1550 | u16 ddp_seg_len; |
1551 | |
1552 | ddp_seg_len = ntohs(*(__be16 *)pkt); |
1553 | if (ddp_seg_len) { |
1554 | *copy_len = 2; |
1555 | hdr->hdrct = DDP_LEN_FLAG; |
1556 | if (pkt[2] & 0x80) { |
1557 | *is_tagged = 1; |
1558 | if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) { |
1559 | *copy_len += TERM_DDP_LEN_TAGGED; |
1560 | hdr->hdrct |= DDP_HDR_FLAG; |
1561 | } |
1562 | } else { |
1563 | if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) { |
1564 | *copy_len += TERM_DDP_LEN_UNTAGGED; |
1565 | hdr->hdrct |= DDP_HDR_FLAG; |
1566 | } |
1567 | if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) && |
1568 | ((pkt[3] & RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) { |
1569 | *copy_len += TERM_RDMA_LEN; |
1570 | hdr->hdrct |= RDMA_HDR_FLAG; |
1571 | } |
1572 | } |
1573 | } |
1574 | } |
1575 | |
1576 | /** |
1577 | * irdma_bld_terminate_hdr - build terminate message header |
1578 | * @qp: qp associated with received terminate AE |
1579 | * @info: the struct contiaing AE information |
1580 | */ |
1581 | static int irdma_bld_terminate_hdr(struct irdma_sc_qp *qp, |
1582 | struct irdma_aeqe_info *info) |
1583 | { |
1584 | u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; |
1585 | int copy_len = 0; |
1586 | u8 is_tagged = 0; |
1587 | u32 opcode; |
1588 | struct irdma_terminate_hdr *termhdr; |
1589 | |
1590 | termhdr = (struct irdma_terminate_hdr *)qp->q2_buf; |
1591 | memset(termhdr, 0, Q2_BAD_FRAME_OFFSET); |
1592 | |
1593 | if (info->q2_data_written) { |
1594 | pkt = irdma_locate_mpa(pkt); |
1595 | irdma_bld_termhdr_ddp_rdma(pkt, hdr: termhdr, copy_len: ©_len, is_tagged: &is_tagged); |
1596 | } |
1597 | |
1598 | opcode = irdma_iwarp_opcode(info, pkt); |
1599 | qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; |
1600 | qp->sq_flush_code = info->sq; |
1601 | qp->rq_flush_code = info->rq; |
1602 | |
1603 | switch (info->ae_id) { |
1604 | case IRDMA_AE_AMP_UNALLOCATED_STAG: |
1605 | qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; |
1606 | if (opcode == IRDMA_OP_TYPE_RDMA_WRITE) |
1607 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_PROT_ERR, |
1608 | layer_etype: (LAYER_DDP << 4) | DDP_TAGGED_BUF, |
1609 | err: DDP_TAGGED_INV_STAG); |
1610 | else |
1611 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_ACCESS_ERR, |
1612 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, |
1613 | err: RDMAP_INV_STAG); |
1614 | break; |
1615 | case IRDMA_AE_AMP_BOUNDS_VIOLATION: |
1616 | qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; |
1617 | if (info->q2_data_written) |
1618 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_PROT_ERR, |
1619 | layer_etype: (LAYER_DDP << 4) | DDP_TAGGED_BUF, |
1620 | err: DDP_TAGGED_BOUNDS); |
1621 | else |
1622 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_ACCESS_ERR, |
1623 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, |
1624 | err: RDMAP_INV_BOUNDS); |
1625 | break; |
1626 | case IRDMA_AE_AMP_BAD_PD: |
1627 | switch (opcode) { |
1628 | case IRDMA_OP_TYPE_RDMA_WRITE: |
1629 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_PROT_ERR, |
1630 | layer_etype: (LAYER_DDP << 4) | DDP_TAGGED_BUF, |
1631 | err: DDP_TAGGED_UNASSOC_STAG); |
1632 | break; |
1633 | case IRDMA_OP_TYPE_SEND_INV: |
1634 | case IRDMA_OP_TYPE_SEND_SOL_INV: |
1635 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_ACCESS_ERR, |
1636 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, |
1637 | err: RDMAP_CANT_INV_STAG); |
1638 | break; |
1639 | default: |
1640 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_ACCESS_ERR, |
1641 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, |
1642 | err: RDMAP_UNASSOC_STAG); |
1643 | } |
1644 | break; |
1645 | case IRDMA_AE_AMP_INVALID_STAG: |
1646 | qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; |
1647 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_ACCESS_ERR, |
1648 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, |
1649 | err: RDMAP_INV_STAG); |
1650 | break; |
1651 | case IRDMA_AE_AMP_BAD_QP: |
1652 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_LOC_QP_OP_ERR, |
1653 | layer_etype: (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, |
1654 | err: DDP_UNTAGGED_INV_QN); |
1655 | break; |
1656 | case IRDMA_AE_AMP_BAD_STAG_KEY: |
1657 | case IRDMA_AE_AMP_BAD_STAG_INDEX: |
1658 | qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; |
1659 | switch (opcode) { |
1660 | case IRDMA_OP_TYPE_SEND_INV: |
1661 | case IRDMA_OP_TYPE_SEND_SOL_INV: |
1662 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_OP_ERR, |
1663 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, |
1664 | err: RDMAP_CANT_INV_STAG); |
1665 | break; |
1666 | default: |
1667 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_ACCESS_ERR, |
1668 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, |
1669 | err: RDMAP_INV_STAG); |
1670 | } |
1671 | break; |
1672 | case IRDMA_AE_AMP_RIGHTS_VIOLATION: |
1673 | case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: |
1674 | case IRDMA_AE_PRIV_OPERATION_DENIED: |
1675 | qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; |
1676 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_ACCESS_ERR, |
1677 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, |
1678 | err: RDMAP_ACCESS); |
1679 | break; |
1680 | case IRDMA_AE_AMP_TO_WRAP: |
1681 | qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; |
1682 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_ACCESS_ERR, |
1683 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, |
1684 | err: RDMAP_TO_WRAP); |
1685 | break; |
1686 | case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: |
1687 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_GENERAL_ERR, |
1688 | layer_etype: (LAYER_MPA << 4) | DDP_LLP, err: MPA_CRC); |
1689 | break; |
1690 | case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: |
1691 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_LOC_LEN_ERR, |
1692 | layer_etype: (LAYER_DDP << 4) | DDP_CATASTROPHIC, |
1693 | err: DDP_CATASTROPHIC_LOCAL); |
1694 | break; |
1695 | case IRDMA_AE_LCE_QP_CATASTROPHIC: |
1696 | case IRDMA_AE_DDP_NO_L_BIT: |
1697 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_FATAL_ERR, |
1698 | layer_etype: (LAYER_DDP << 4) | DDP_CATASTROPHIC, |
1699 | err: DDP_CATASTROPHIC_LOCAL); |
1700 | break; |
1701 | case IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN: |
1702 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_GENERAL_ERR, |
1703 | layer_etype: (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, |
1704 | err: DDP_UNTAGGED_INV_MSN_RANGE); |
1705 | break; |
1706 | case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: |
1707 | qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; |
1708 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_LOC_LEN_ERR, |
1709 | layer_etype: (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, |
1710 | err: DDP_UNTAGGED_INV_TOO_LONG); |
1711 | break; |
1712 | case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION: |
1713 | if (is_tagged) |
1714 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_GENERAL_ERR, |
1715 | layer_etype: (LAYER_DDP << 4) | DDP_TAGGED_BUF, |
1716 | err: DDP_TAGGED_INV_DDP_VER); |
1717 | else |
1718 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_GENERAL_ERR, |
1719 | layer_etype: (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, |
1720 | err: DDP_UNTAGGED_INV_DDP_VER); |
1721 | break; |
1722 | case IRDMA_AE_DDP_UBE_INVALID_MO: |
1723 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_GENERAL_ERR, |
1724 | layer_etype: (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, |
1725 | err: DDP_UNTAGGED_INV_MO); |
1726 | break; |
1727 | case IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: |
1728 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_REM_OP_ERR, |
1729 | layer_etype: (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, |
1730 | err: DDP_UNTAGGED_INV_MSN_NO_BUF); |
1731 | break; |
1732 | case IRDMA_AE_DDP_UBE_INVALID_QN: |
1733 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_GENERAL_ERR, |
1734 | layer_etype: (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, |
1735 | err: DDP_UNTAGGED_INV_QN); |
1736 | break; |
1737 | case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION: |
1738 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_GENERAL_ERR, |
1739 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, |
1740 | err: RDMAP_INV_RDMAP_VER); |
1741 | break; |
1742 | default: |
1743 | irdma_bld_termhdr_ctrl(qp, hdr: termhdr, opcode: FLUSH_FATAL_ERR, |
1744 | layer_etype: (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, |
1745 | err: RDMAP_UNSPECIFIED); |
1746 | break; |
1747 | } |
1748 | |
1749 | if (copy_len) |
1750 | memcpy(termhdr + 1, pkt, copy_len); |
1751 | |
1752 | return sizeof(struct irdma_terminate_hdr) + copy_len; |
1753 | } |
1754 | |
1755 | /** |
1756 | * irdma_terminate_send_fin() - Send fin for terminate message |
1757 | * @qp: qp associated with received terminate AE |
1758 | */ |
1759 | void irdma_terminate_send_fin(struct irdma_sc_qp *qp) |
1760 | { |
1761 | irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE, |
1762 | IRDMAQP_TERM_SEND_FIN_ONLY, term_len: 0); |
1763 | } |
1764 | |
1765 | /** |
1766 | * irdma_terminate_connection() - Bad AE and send terminate to remote QP |
1767 | * @qp: qp associated with received terminate AE |
1768 | * @info: the struct contiaing AE information |
1769 | */ |
1770 | void irdma_terminate_connection(struct irdma_sc_qp *qp, |
1771 | struct irdma_aeqe_info *info) |
1772 | { |
1773 | u8 termlen = 0; |
1774 | |
1775 | if (qp->term_flags & IRDMA_TERM_SENT) |
1776 | return; |
1777 | |
1778 | termlen = irdma_bld_terminate_hdr(qp, info); |
1779 | irdma_terminate_start_timer(qp); |
1780 | qp->term_flags |= IRDMA_TERM_SENT; |
1781 | irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE, |
1782 | IRDMAQP_TERM_SEND_TERM_ONLY, term_len: termlen); |
1783 | } |
1784 | |
1785 | /** |
1786 | * irdma_terminate_received - handle terminate received AE |
1787 | * @qp: qp associated with received terminate AE |
1788 | * @info: the struct contiaing AE information |
1789 | */ |
1790 | void irdma_terminate_received(struct irdma_sc_qp *qp, |
1791 | struct irdma_aeqe_info *info) |
1792 | { |
1793 | u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; |
1794 | __be32 *mpa; |
1795 | u8 ddp_ctl; |
1796 | u8 rdma_ctl; |
1797 | u16 aeq_id = 0; |
1798 | struct irdma_terminate_hdr *termhdr; |
1799 | |
1800 | mpa = (__be32 *)irdma_locate_mpa(pkt); |
1801 | if (info->q2_data_written) { |
1802 | /* did not validate the frame - do it now */ |
1803 | ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff; |
1804 | rdma_ctl = ntohl(mpa[0]) & 0xff; |
1805 | if ((ddp_ctl & 0xc0) != 0x40) |
1806 | aeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC; |
1807 | else if ((ddp_ctl & 0x03) != 1) |
1808 | aeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION; |
1809 | else if (ntohl(mpa[2]) != 2) |
1810 | aeq_id = IRDMA_AE_DDP_UBE_INVALID_QN; |
1811 | else if (ntohl(mpa[3]) != 1) |
1812 | aeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN; |
1813 | else if (ntohl(mpa[4]) != 0) |
1814 | aeq_id = IRDMA_AE_DDP_UBE_INVALID_MO; |
1815 | else if ((rdma_ctl & 0xc0) != 0x40) |
1816 | aeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION; |
1817 | |
1818 | info->ae_id = aeq_id; |
1819 | if (info->ae_id) { |
1820 | /* Bad terminate recvd - send back a terminate */ |
1821 | irdma_terminate_connection(qp, info); |
1822 | return; |
1823 | } |
1824 | } |
1825 | |
1826 | qp->term_flags |= IRDMA_TERM_RCVD; |
1827 | qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; |
1828 | termhdr = (struct irdma_terminate_hdr *)&mpa[5]; |
1829 | if (termhdr->layer_etype == RDMAP_REMOTE_PROT || |
1830 | termhdr->layer_etype == RDMAP_REMOTE_OP) { |
1831 | irdma_terminate_done(qp, timeout_occurred: 0); |
1832 | } else { |
1833 | irdma_terminate_start_timer(qp); |
1834 | irdma_terminate_send_fin(qp); |
1835 | } |
1836 | } |
1837 | |
1838 | static int irdma_null_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri) |
1839 | { |
1840 | return 0; |
1841 | } |
1842 | |
1843 | static void irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri) |
1844 | { |
1845 | /* do nothing */ |
1846 | } |
1847 | |
1848 | static void irdma_null_ws_reset(struct irdma_sc_vsi *vsi) |
1849 | { |
1850 | /* do nothing */ |
1851 | } |
1852 | |
1853 | /** |
1854 | * irdma_sc_vsi_init - Init the vsi structure |
1855 | * @vsi: pointer to vsi structure to initialize |
1856 | * @info: the info used to initialize the vsi struct |
1857 | */ |
1858 | void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi, |
1859 | struct irdma_vsi_init_info *info) |
1860 | { |
1861 | int i; |
1862 | |
1863 | vsi->dev = info->dev; |
1864 | vsi->back_vsi = info->back_vsi; |
1865 | vsi->register_qset = info->register_qset; |
1866 | vsi->unregister_qset = info->unregister_qset; |
1867 | vsi->mtu = info->params->mtu; |
1868 | vsi->exception_lan_q = info->exception_lan_q; |
1869 | vsi->vsi_idx = info->pf_data_vsi_num; |
1870 | |
1871 | irdma_set_qos_info(vsi, l2p: info->params); |
1872 | for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { |
1873 | mutex_init(&vsi->qos[i].qos_mutex); |
1874 | INIT_LIST_HEAD(list: &vsi->qos[i].qplist); |
1875 | } |
1876 | if (vsi->register_qset) { |
1877 | vsi->dev->ws_add = irdma_ws_add; |
1878 | vsi->dev->ws_remove = irdma_ws_remove; |
1879 | vsi->dev->ws_reset = irdma_ws_reset; |
1880 | } else { |
1881 | vsi->dev->ws_add = irdma_null_ws_add; |
1882 | vsi->dev->ws_remove = irdma_null_ws_remove; |
1883 | vsi->dev->ws_reset = irdma_null_ws_reset; |
1884 | } |
1885 | } |
1886 | |
1887 | /** |
1888 | * irdma_get_stats_idx - Return stats index |
1889 | * @vsi: pointer to the vsi |
1890 | */ |
1891 | static u8 irdma_get_stats_idx(struct irdma_sc_vsi *vsi) |
1892 | { |
1893 | struct irdma_stats_inst_info stats_info = {}; |
1894 | struct irdma_sc_dev *dev = vsi->dev; |
1895 | u8 i; |
1896 | |
1897 | if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { |
1898 | if (!irdma_cqp_stats_inst_cmd(vsi, cmd: IRDMA_OP_STATS_ALLOCATE, |
1899 | stats_info: &stats_info)) |
1900 | return stats_info.stats_idx; |
1901 | } |
1902 | |
1903 | for (i = 0; i < IRDMA_MAX_STATS_COUNT_GEN_1; i++) { |
1904 | if (!dev->stats_idx_array[i]) { |
1905 | dev->stats_idx_array[i] = true; |
1906 | return i; |
1907 | } |
1908 | } |
1909 | |
1910 | return IRDMA_INVALID_STATS_IDX; |
1911 | } |
1912 | |
1913 | /** |
1914 | * irdma_hw_stats_init_gen1 - Initialize stat reg table used for gen1 |
1915 | * @vsi: vsi structure where hw_regs are set |
1916 | * |
1917 | * Populate the HW stats table |
1918 | */ |
1919 | static void irdma_hw_stats_init_gen1(struct irdma_sc_vsi *vsi) |
1920 | { |
1921 | struct irdma_sc_dev *dev = vsi->dev; |
1922 | const struct irdma_hw_stat_map *map; |
1923 | u64 *stat_reg = vsi->hw_stats_regs; |
1924 | u64 *regs = dev->hw_stats_regs; |
1925 | u16 i, stats_reg_set = vsi->stats_idx; |
1926 | |
1927 | map = dev->hw_stats_map; |
1928 | |
1929 | /* First 4 stat instances are reserved for port level statistics. */ |
1930 | stats_reg_set += vsi->stats_inst_alloc ? IRDMA_FIRST_NON_PF_STAT : 0; |
1931 | |
1932 | for (i = 0; i < dev->hw_attrs.max_stat_idx; i++) { |
1933 | if (map[i].bitmask <= IRDMA_MAX_STATS_32) |
1934 | stat_reg[i] = regs[i] + stats_reg_set * sizeof(u32); |
1935 | else |
1936 | stat_reg[i] = regs[i] + stats_reg_set * sizeof(u64); |
1937 | } |
1938 | } |
1939 | |
1940 | /** |
1941 | * irdma_vsi_stats_init - Initialize the vsi statistics |
1942 | * @vsi: pointer to the vsi structure |
1943 | * @info: The info structure used for initialization |
1944 | */ |
1945 | int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi, |
1946 | struct irdma_vsi_stats_info *info) |
1947 | { |
1948 | struct irdma_dma_mem *stats_buff_mem; |
1949 | |
1950 | vsi->pestat = info->pestat; |
1951 | vsi->pestat->hw = vsi->dev->hw; |
1952 | vsi->pestat->vsi = vsi; |
1953 | stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem; |
1954 | stats_buff_mem->size = ALIGN(IRDMA_GATHER_STATS_BUF_SIZE * 2, 1); |
1955 | stats_buff_mem->va = dma_alloc_coherent(dev: vsi->pestat->hw->device, |
1956 | size: stats_buff_mem->size, |
1957 | dma_handle: &stats_buff_mem->pa, |
1958 | GFP_KERNEL); |
1959 | if (!stats_buff_mem->va) |
1960 | return -ENOMEM; |
1961 | |
1962 | vsi->pestat->gather_info.gather_stats_va = stats_buff_mem->va; |
1963 | vsi->pestat->gather_info.last_gather_stats_va = |
1964 | (void *)((uintptr_t)stats_buff_mem->va + |
1965 | IRDMA_GATHER_STATS_BUF_SIZE); |
1966 | |
1967 | irdma_hw_stats_start_timer(vsi); |
1968 | |
1969 | /* when stat allocation is not required default to fcn_id. */ |
1970 | vsi->stats_idx = info->fcn_id; |
1971 | if (info->alloc_stats_inst) { |
1972 | u8 stats_idx = irdma_get_stats_idx(vsi); |
1973 | |
1974 | if (stats_idx != IRDMA_INVALID_STATS_IDX) { |
1975 | vsi->stats_inst_alloc = true; |
1976 | vsi->stats_idx = stats_idx; |
1977 | vsi->pestat->gather_info.use_stats_inst = true; |
1978 | vsi->pestat->gather_info.stats_inst_index = stats_idx; |
1979 | } |
1980 | } |
1981 | |
1982 | if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) |
1983 | irdma_hw_stats_init_gen1(vsi); |
1984 | |
1985 | return 0; |
1986 | } |
1987 | |
1988 | /** |
1989 | * irdma_vsi_stats_free - Free the vsi stats |
1990 | * @vsi: pointer to the vsi structure |
1991 | */ |
1992 | void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi) |
1993 | { |
1994 | struct irdma_stats_inst_info stats_info = {}; |
1995 | struct irdma_sc_dev *dev = vsi->dev; |
1996 | u8 stats_idx = vsi->stats_idx; |
1997 | |
1998 | if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { |
1999 | if (vsi->stats_inst_alloc) { |
2000 | stats_info.stats_idx = vsi->stats_idx; |
2001 | irdma_cqp_stats_inst_cmd(vsi, cmd: IRDMA_OP_STATS_FREE, |
2002 | stats_info: &stats_info); |
2003 | } |
2004 | } else { |
2005 | if (vsi->stats_inst_alloc && |
2006 | stats_idx < vsi->dev->hw_attrs.max_stat_inst) |
2007 | vsi->dev->stats_idx_array[stats_idx] = false; |
2008 | } |
2009 | |
2010 | if (!vsi->pestat) |
2011 | return; |
2012 | irdma_hw_stats_stop_timer(vsi); |
2013 | dma_free_coherent(dev: vsi->pestat->hw->device, |
2014 | size: vsi->pestat->gather_info.stats_buff_mem.size, |
2015 | cpu_addr: vsi->pestat->gather_info.stats_buff_mem.va, |
2016 | dma_handle: vsi->pestat->gather_info.stats_buff_mem.pa); |
2017 | vsi->pestat->gather_info.stats_buff_mem.va = NULL; |
2018 | } |
2019 | |
2020 | /** |
2021 | * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size |
2022 | * @wqsize: size of the wq (sq, rq) to encoded_size |
2023 | * @queue_type: queue type selected for the calculation algorithm |
2024 | */ |
2025 | u8 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type) |
2026 | { |
2027 | u8 encoded_size = 0; |
2028 | |
2029 | /* cqp sq's hw coded value starts from 1 for size of 4 |
2030 | * while it starts from 0 for qp' wq's. |
2031 | */ |
2032 | if (queue_type == IRDMA_QUEUE_TYPE_CQP) |
2033 | encoded_size = 1; |
2034 | wqsize >>= 2; |
2035 | while (wqsize >>= 1) |
2036 | encoded_size++; |
2037 | |
2038 | return encoded_size; |
2039 | } |
2040 | |
2041 | /** |
2042 | * irdma_sc_gather_stats - collect the statistics |
2043 | * @cqp: struct for cqp hw |
2044 | * @info: gather stats info structure |
2045 | * @scratch: u64 saved to be used during cqp completion |
2046 | */ |
2047 | static int irdma_sc_gather_stats(struct irdma_sc_cqp *cqp, |
2048 | struct irdma_stats_gather_info *info, |
2049 | u64 scratch) |
2050 | { |
2051 | __le64 *wqe; |
2052 | u64 temp; |
2053 | |
2054 | if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE) |
2055 | return -ENOMEM; |
2056 | |
2057 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2058 | if (!wqe) |
2059 | return -ENOMEM; |
2060 | |
2061 | set_64bit_val(wqe_words: wqe, byte_index: 40, |
2062 | FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fcn_index)); |
2063 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: info->stats_buff_mem.pa); |
2064 | |
2065 | temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | |
2066 | FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info->use_stats_inst) | |
2067 | FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, |
2068 | info->stats_inst_index) | |
2069 | FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX, |
2070 | info->use_hmc_fcn_index) | |
2071 | FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_GATHER_STATS); |
2072 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2073 | |
2074 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: temp); |
2075 | |
2076 | print_hex_dump_debug("STATS: GATHER_STATS WQE" , DUMP_PREFIX_OFFSET, |
2077 | 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2078 | |
2079 | irdma_sc_cqp_post_sq(cqp); |
2080 | ibdev_dbg(to_ibdev(cqp->dev), |
2081 | "STATS: CQP SQ head 0x%x tail 0x%x size 0x%x\n" , |
2082 | cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); |
2083 | |
2084 | return 0; |
2085 | } |
2086 | |
2087 | /** |
2088 | * irdma_sc_manage_stats_inst - allocate or free stats instance |
2089 | * @cqp: struct for cqp hw |
2090 | * @info: stats info structure |
2091 | * @alloc: alloc vs. delete flag |
2092 | * @scratch: u64 saved to be used during cqp completion |
2093 | */ |
2094 | static int irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp, |
2095 | struct irdma_stats_inst_info *info, |
2096 | bool alloc, u64 scratch) |
2097 | { |
2098 | __le64 *wqe; |
2099 | u64 temp; |
2100 | |
2101 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2102 | if (!wqe) |
2103 | return -ENOMEM; |
2104 | |
2105 | set_64bit_val(wqe_words: wqe, byte_index: 40, |
2106 | FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fn_id)); |
2107 | temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | |
2108 | FIELD_PREP(IRDMA_CQPSQ_STATS_ALLOC_INST, alloc) | |
2109 | FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX, |
2110 | info->use_hmc_fcn_index) | |
2111 | FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info->stats_idx) | |
2112 | FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_MANAGE_STATS); |
2113 | |
2114 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2115 | |
2116 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: temp); |
2117 | |
2118 | print_hex_dump_debug("WQE: MANAGE_STATS WQE" , DUMP_PREFIX_OFFSET, 16, |
2119 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2120 | |
2121 | irdma_sc_cqp_post_sq(cqp); |
2122 | return 0; |
2123 | } |
2124 | |
2125 | /** |
2126 | * irdma_sc_set_up_map - set the up map table |
2127 | * @cqp: struct for cqp hw |
2128 | * @info: User priority map info |
2129 | * @scratch: u64 saved to be used during cqp completion |
2130 | */ |
2131 | static int irdma_sc_set_up_map(struct irdma_sc_cqp *cqp, |
2132 | struct irdma_up_info *info, u64 scratch) |
2133 | { |
2134 | __le64 *wqe; |
2135 | u64 temp = 0; |
2136 | int i; |
2137 | |
2138 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2139 | if (!wqe) |
2140 | return -ENOMEM; |
2141 | |
2142 | for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) |
2143 | temp |= (u64)info->map[i] << (i * 8); |
2144 | |
2145 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: temp); |
2146 | set_64bit_val(wqe_words: wqe, byte_index: 40, |
2147 | FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info->cnp_up_override) | |
2148 | FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info->hmc_fcn_idx)); |
2149 | |
2150 | temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) | |
2151 | FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info->use_vlan) | |
2152 | FIELD_PREP(IRDMA_CQPSQ_UP_USEOVERRIDE, |
2153 | info->use_cnp_up_override) | |
2154 | FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_UP_MAP); |
2155 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2156 | |
2157 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: temp); |
2158 | |
2159 | print_hex_dump_debug("WQE: UPMAP WQE" , DUMP_PREFIX_OFFSET, 16, 8, wqe, |
2160 | IRDMA_CQP_WQE_SIZE * 8, false); |
2161 | irdma_sc_cqp_post_sq(cqp); |
2162 | |
2163 | return 0; |
2164 | } |
2165 | |
2166 | /** |
2167 | * irdma_sc_manage_ws_node - create/modify/destroy WS node |
2168 | * @cqp: struct for cqp hw |
2169 | * @info: node info structure |
2170 | * @node_op: 0 for add 1 for modify, 2 for delete |
2171 | * @scratch: u64 saved to be used during cqp completion |
2172 | */ |
2173 | static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp, |
2174 | struct irdma_ws_node_info *info, |
2175 | enum irdma_ws_node_op node_op, u64 scratch) |
2176 | { |
2177 | __le64 *wqe; |
2178 | u64 temp = 0; |
2179 | |
2180 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2181 | if (!wqe) |
2182 | return -ENOMEM; |
2183 | |
2184 | set_64bit_val(wqe_words: wqe, byte_index: 32, |
2185 | FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info->vsi) | |
2186 | FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info->weight)); |
2187 | |
2188 | temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) | |
2189 | FIELD_PREP(IRDMA_CQPSQ_WS_NODEOP, node_op) | |
2190 | FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info->enable) | |
2191 | FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info->type_leaf) | |
2192 | FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info->prio_type) | |
2193 | FIELD_PREP(IRDMA_CQPSQ_WS_TC, info->tc) | |
2194 | FIELD_PREP(IRDMA_CQPSQ_WS_OP, IRDMA_CQP_OP_WORK_SCHED_NODE) | |
2195 | FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info->parent_id) | |
2196 | FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info->id); |
2197 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2198 | |
2199 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: temp); |
2200 | |
2201 | print_hex_dump_debug("WQE: MANAGE_WS WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
2202 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2203 | irdma_sc_cqp_post_sq(cqp); |
2204 | |
2205 | return 0; |
2206 | } |
2207 | |
2208 | /** |
2209 | * irdma_sc_qp_flush_wqes - flush qp's wqe |
2210 | * @qp: sc qp |
2211 | * @info: dlush information |
2212 | * @scratch: u64 saved to be used during cqp completion |
2213 | * @post_sq: flag for cqp db to ring |
2214 | */ |
2215 | int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, |
2216 | struct irdma_qp_flush_info *info, u64 scratch, |
2217 | bool post_sq) |
2218 | { |
2219 | u64 temp = 0; |
2220 | __le64 *wqe; |
2221 | struct irdma_sc_cqp *cqp; |
2222 | u64 hdr; |
2223 | bool flush_sq = false, flush_rq = false; |
2224 | |
2225 | if (info->rq && !qp->flush_rq) |
2226 | flush_rq = true; |
2227 | if (info->sq && !qp->flush_sq) |
2228 | flush_sq = true; |
2229 | qp->flush_sq |= flush_sq; |
2230 | qp->flush_rq |= flush_rq; |
2231 | |
2232 | if (!flush_sq && !flush_rq) { |
2233 | ibdev_dbg(to_ibdev(qp->dev), |
2234 | "CQP: Additional flush request ignored for qp %x\n" , |
2235 | qp->qp_uk.qp_id); |
2236 | return -EALREADY; |
2237 | } |
2238 | |
2239 | cqp = qp->pd->dev->cqp; |
2240 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2241 | if (!wqe) |
2242 | return -ENOMEM; |
2243 | |
2244 | if (info->userflushcode) { |
2245 | if (flush_rq) |
2246 | temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMNERR, |
2247 | info->rq_minor_code) | |
2248 | FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMJERR, |
2249 | info->rq_major_code); |
2250 | if (flush_sq) |
2251 | temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMNERR, |
2252 | info->sq_minor_code) | |
2253 | FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMJERR, |
2254 | info->sq_major_code); |
2255 | } |
2256 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: temp); |
2257 | |
2258 | temp = (info->generate_ae) ? |
2259 | info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE, |
2260 | info->ae_src) : 0; |
2261 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: temp); |
2262 | |
2263 | hdr = qp->qp_uk.qp_id | |
2264 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_FLUSH_WQES) | |
2265 | FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) | |
2266 | FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info->userflushcode) | |
2267 | FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHSQ, flush_sq) | |
2268 | FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHRQ, flush_rq) | |
2269 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
2270 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2271 | |
2272 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
2273 | |
2274 | print_hex_dump_debug("WQE: QP_FLUSH WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
2275 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2276 | if (post_sq) |
2277 | irdma_sc_cqp_post_sq(cqp); |
2278 | |
2279 | return 0; |
2280 | } |
2281 | |
2282 | /** |
2283 | * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP |
2284 | * @qp: sc qp |
2285 | * @info: gen ae information |
2286 | * @scratch: u64 saved to be used during cqp completion |
2287 | * @post_sq: flag for cqp db to ring |
2288 | */ |
2289 | static int irdma_sc_gen_ae(struct irdma_sc_qp *qp, |
2290 | struct irdma_gen_ae_info *info, u64 scratch, |
2291 | bool post_sq) |
2292 | { |
2293 | u64 temp; |
2294 | __le64 *wqe; |
2295 | struct irdma_sc_cqp *cqp; |
2296 | u64 hdr; |
2297 | |
2298 | cqp = qp->pd->dev->cqp; |
2299 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2300 | if (!wqe) |
2301 | return -ENOMEM; |
2302 | |
2303 | temp = info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE, |
2304 | info->ae_src); |
2305 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: temp); |
2306 | |
2307 | hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE, |
2308 | IRDMA_CQP_OP_GEN_AE) | |
2309 | FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, 1) | |
2310 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
2311 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2312 | |
2313 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
2314 | |
2315 | print_hex_dump_debug("WQE: GEN_AE WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
2316 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2317 | if (post_sq) |
2318 | irdma_sc_cqp_post_sq(cqp); |
2319 | |
2320 | return 0; |
2321 | } |
2322 | |
2323 | /*** irdma_sc_qp_upload_context - upload qp's context |
2324 | * @dev: sc device struct |
2325 | * @info: upload context info ptr for return |
2326 | * @scratch: u64 saved to be used during cqp completion |
2327 | * @post_sq: flag for cqp db to ring |
2328 | */ |
2329 | static int irdma_sc_qp_upload_context(struct irdma_sc_dev *dev, |
2330 | struct irdma_upload_context_info *info, |
2331 | u64 scratch, bool post_sq) |
2332 | { |
2333 | __le64 *wqe; |
2334 | struct irdma_sc_cqp *cqp; |
2335 | u64 hdr; |
2336 | |
2337 | cqp = dev->cqp; |
2338 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2339 | if (!wqe) |
2340 | return -ENOMEM; |
2341 | |
2342 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: info->buf_pa); |
2343 | |
2344 | hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info->qp_id) | |
2345 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPLOAD_CONTEXT) | |
2346 | FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info->qp_type) | |
2347 | FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info->raw_format) | |
2348 | FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info->freeze_qp) | |
2349 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
2350 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2351 | |
2352 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
2353 | |
2354 | print_hex_dump_debug("WQE: QP_UPLOAD_CTX WQE" , DUMP_PREFIX_OFFSET, 16, |
2355 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2356 | if (post_sq) |
2357 | irdma_sc_cqp_post_sq(cqp); |
2358 | |
2359 | return 0; |
2360 | } |
2361 | |
2362 | /** |
2363 | * irdma_sc_manage_push_page - Handle push page |
2364 | * @cqp: struct for cqp hw |
2365 | * @info: push page info |
2366 | * @scratch: u64 saved to be used during cqp completion |
2367 | * @post_sq: flag for cqp db to ring |
2368 | */ |
2369 | static int irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp, |
2370 | struct irdma_cqp_manage_push_page_info *info, |
2371 | u64 scratch, bool post_sq) |
2372 | { |
2373 | __le64 *wqe; |
2374 | u64 hdr; |
2375 | |
2376 | if (info->free_page && |
2377 | info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages) |
2378 | return -EINVAL; |
2379 | |
2380 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2381 | if (!wqe) |
2382 | return -ENOMEM; |
2383 | |
2384 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: info->qs_handle); |
2385 | hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info->push_idx) | |
2386 | FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info->push_page_type) | |
2387 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_PUSH_PAGES) | |
2388 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | |
2389 | FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info->free_page); |
2390 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2391 | |
2392 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
2393 | |
2394 | print_hex_dump_debug("WQE: MANAGE_PUSH_PAGES WQE" , DUMP_PREFIX_OFFSET, |
2395 | 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2396 | if (post_sq) |
2397 | irdma_sc_cqp_post_sq(cqp); |
2398 | |
2399 | return 0; |
2400 | } |
2401 | |
2402 | /** |
2403 | * irdma_sc_suspend_qp - suspend qp for param change |
2404 | * @cqp: struct for cqp hw |
2405 | * @qp: sc qp struct |
2406 | * @scratch: u64 saved to be used during cqp completion |
2407 | */ |
2408 | static int irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, |
2409 | u64 scratch) |
2410 | { |
2411 | u64 hdr; |
2412 | __le64 *wqe; |
2413 | |
2414 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2415 | if (!wqe) |
2416 | return -ENOMEM; |
2417 | |
2418 | hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp->qp_uk.qp_id) | |
2419 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_SUSPEND_QP) | |
2420 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
2421 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2422 | |
2423 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
2424 | |
2425 | print_hex_dump_debug("WQE: SUSPEND_QP WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
2426 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2427 | irdma_sc_cqp_post_sq(cqp); |
2428 | |
2429 | return 0; |
2430 | } |
2431 | |
2432 | /** |
2433 | * irdma_sc_resume_qp - resume qp after suspend |
2434 | * @cqp: struct for cqp hw |
2435 | * @qp: sc qp struct |
2436 | * @scratch: u64 saved to be used during cqp completion |
2437 | */ |
2438 | static int irdma_sc_resume_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, |
2439 | u64 scratch) |
2440 | { |
2441 | u64 hdr; |
2442 | __le64 *wqe; |
2443 | |
2444 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2445 | if (!wqe) |
2446 | return -ENOMEM; |
2447 | |
2448 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
2449 | FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp->qs_handle)); |
2450 | |
2451 | hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp->qp_uk.qp_id) | |
2452 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_RESUME_QP) | |
2453 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
2454 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2455 | |
2456 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
2457 | |
2458 | print_hex_dump_debug("WQE: RESUME_QP WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
2459 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2460 | irdma_sc_cqp_post_sq(cqp); |
2461 | |
2462 | return 0; |
2463 | } |
2464 | |
2465 | /** |
2466 | * irdma_sc_cq_ack - acknowledge completion q |
2467 | * @cq: cq struct |
2468 | */ |
2469 | static inline void irdma_sc_cq_ack(struct irdma_sc_cq *cq) |
2470 | { |
2471 | writel(val: cq->cq_uk.cq_id, addr: cq->cq_uk.cq_ack_db); |
2472 | } |
2473 | |
2474 | /** |
2475 | * irdma_sc_cq_init - initialize completion q |
2476 | * @cq: cq struct |
2477 | * @info: cq initialization info |
2478 | */ |
2479 | int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info) |
2480 | { |
2481 | u32 pble_obj_cnt; |
2482 | |
2483 | pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; |
2484 | if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) |
2485 | return -EINVAL; |
2486 | |
2487 | cq->cq_pa = info->cq_base_pa; |
2488 | cq->dev = info->dev; |
2489 | cq->ceq_id = info->ceq_id; |
2490 | info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db; |
2491 | info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db; |
2492 | irdma_uk_cq_init(cq: &cq->cq_uk, info: &info->cq_uk_init_info); |
2493 | |
2494 | cq->virtual_map = info->virtual_map; |
2495 | cq->pbl_chunk_size = info->pbl_chunk_size; |
2496 | cq->ceqe_mask = info->ceqe_mask; |
2497 | cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP; |
2498 | cq->shadow_area_pa = info->shadow_area_pa; |
2499 | cq->shadow_read_threshold = info->shadow_read_threshold; |
2500 | cq->ceq_id_valid = info->ceq_id_valid; |
2501 | cq->tph_en = info->tph_en; |
2502 | cq->tph_val = info->tph_val; |
2503 | cq->first_pm_pbl_idx = info->first_pm_pbl_idx; |
2504 | cq->vsi = info->vsi; |
2505 | |
2506 | return 0; |
2507 | } |
2508 | |
2509 | /** |
2510 | * irdma_sc_cq_create - create completion q |
2511 | * @cq: cq struct |
2512 | * @scratch: u64 saved to be used during cqp completion |
2513 | * @check_overflow: flag for overflow check |
2514 | * @post_sq: flag for cqp db to ring |
2515 | */ |
2516 | static int irdma_sc_cq_create(struct irdma_sc_cq *cq, u64 scratch, |
2517 | bool check_overflow, bool post_sq) |
2518 | { |
2519 | __le64 *wqe; |
2520 | struct irdma_sc_cqp *cqp; |
2521 | u64 hdr; |
2522 | struct irdma_sc_ceq *ceq; |
2523 | int ret_code = 0; |
2524 | |
2525 | cqp = cq->dev->cqp; |
2526 | if (cq->cq_uk.cq_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt) |
2527 | return -EINVAL; |
2528 | |
2529 | if (cq->ceq_id >= cq->dev->hmc_fpm_misc.max_ceqs) |
2530 | return -EINVAL; |
2531 | |
2532 | ceq = cq->dev->ceq[cq->ceq_id]; |
2533 | if (ceq && ceq->reg_cq) |
2534 | ret_code = irdma_sc_add_cq_ctx(ceq, cq); |
2535 | |
2536 | if (ret_code) |
2537 | return ret_code; |
2538 | |
2539 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2540 | if (!wqe) { |
2541 | if (ceq && ceq->reg_cq) |
2542 | irdma_sc_remove_cq_ctx(ceq, cq); |
2543 | return -ENOMEM; |
2544 | } |
2545 | |
2546 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: cq->cq_uk.cq_size); |
2547 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: (uintptr_t)cq >> 1); |
2548 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
2549 | FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold)); |
2550 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: (cq->virtual_map ? 0 : cq->cq_pa)); |
2551 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: cq->shadow_area_pa); |
2552 | set_64bit_val(wqe_words: wqe, byte_index: 48, |
2553 | FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0))); |
2554 | set_64bit_val(wqe_words: wqe, byte_index: 56, |
2555 | FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) | |
2556 | FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx)); |
2557 | |
2558 | hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) | |
2559 | FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0), |
2560 | IRDMA_CQPSQ_CQ_CEQID) | |
2561 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) | |
2562 | FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | |
2563 | FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, check_overflow) | |
2564 | FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) | |
2565 | FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | |
2566 | FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) | |
2567 | FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | |
2568 | FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, |
2569 | cq->cq_uk.avoid_mem_cflct) | |
2570 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
2571 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2572 | |
2573 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
2574 | |
2575 | print_hex_dump_debug("WQE: CQ_CREATE WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
2576 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2577 | if (post_sq) |
2578 | irdma_sc_cqp_post_sq(cqp); |
2579 | |
2580 | return 0; |
2581 | } |
2582 | |
2583 | /** |
2584 | * irdma_sc_cq_destroy - destroy completion q |
2585 | * @cq: cq struct |
2586 | * @scratch: u64 saved to be used during cqp completion |
2587 | * @post_sq: flag for cqp db to ring |
2588 | */ |
2589 | int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq) |
2590 | { |
2591 | struct irdma_sc_cqp *cqp; |
2592 | __le64 *wqe; |
2593 | u64 hdr; |
2594 | struct irdma_sc_ceq *ceq; |
2595 | |
2596 | cqp = cq->dev->cqp; |
2597 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2598 | if (!wqe) |
2599 | return -ENOMEM; |
2600 | |
2601 | ceq = cq->dev->ceq[cq->ceq_id]; |
2602 | if (ceq && ceq->reg_cq) |
2603 | irdma_sc_remove_cq_ctx(ceq, cq); |
2604 | |
2605 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: cq->cq_uk.cq_size); |
2606 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: (uintptr_t)cq >> 1); |
2607 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: cq->shadow_area_pa); |
2608 | set_64bit_val(wqe_words: wqe, byte_index: 48, |
2609 | val: (cq->virtual_map ? cq->first_pm_pbl_idx : 0)); |
2610 | |
2611 | hdr = cq->cq_uk.cq_id | |
2612 | FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0), |
2613 | IRDMA_CQPSQ_CQ_CEQID) | |
2614 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) | |
2615 | FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | |
2616 | FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) | |
2617 | FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | |
2618 | FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) | |
2619 | FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | |
2620 | FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq->cq_uk.avoid_mem_cflct) | |
2621 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
2622 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2623 | |
2624 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
2625 | |
2626 | print_hex_dump_debug("WQE: CQ_DESTROY WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
2627 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2628 | if (post_sq) |
2629 | irdma_sc_cqp_post_sq(cqp); |
2630 | |
2631 | return 0; |
2632 | } |
2633 | |
2634 | /** |
2635 | * irdma_sc_cq_resize - set resized cq buffer info |
2636 | * @cq: resized cq |
2637 | * @info: resized cq buffer info |
2638 | */ |
2639 | void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info) |
2640 | { |
2641 | cq->virtual_map = info->virtual_map; |
2642 | cq->cq_pa = info->cq_pa; |
2643 | cq->first_pm_pbl_idx = info->first_pm_pbl_idx; |
2644 | cq->pbl_chunk_size = info->pbl_chunk_size; |
2645 | irdma_uk_cq_resize(cq: &cq->cq_uk, cq_base: info->cq_base, size: info->cq_size); |
2646 | } |
2647 | |
2648 | /** |
2649 | * irdma_sc_cq_modify - modify a Completion Queue |
2650 | * @cq: cq struct |
2651 | * @info: modification info struct |
2652 | * @scratch: u64 saved to be used during cqp completion |
2653 | * @post_sq: flag to post to sq |
2654 | */ |
2655 | static int irdma_sc_cq_modify(struct irdma_sc_cq *cq, |
2656 | struct irdma_modify_cq_info *info, u64 scratch, |
2657 | bool post_sq) |
2658 | { |
2659 | struct irdma_sc_cqp *cqp; |
2660 | __le64 *wqe; |
2661 | u64 hdr; |
2662 | u32 pble_obj_cnt; |
2663 | |
2664 | pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; |
2665 | if (info->cq_resize && info->virtual_map && |
2666 | info->first_pm_pbl_idx >= pble_obj_cnt) |
2667 | return -EINVAL; |
2668 | |
2669 | cqp = cq->dev->cqp; |
2670 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
2671 | if (!wqe) |
2672 | return -ENOMEM; |
2673 | |
2674 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: info->cq_size); |
2675 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: (uintptr_t)cq >> 1); |
2676 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
2677 | FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info->shadow_read_threshold)); |
2678 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: info->cq_pa); |
2679 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: cq->shadow_area_pa); |
2680 | set_64bit_val(wqe_words: wqe, byte_index: 48, val: info->first_pm_pbl_idx); |
2681 | set_64bit_val(wqe_words: wqe, byte_index: 56, |
2682 | FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) | |
2683 | FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx)); |
2684 | |
2685 | hdr = cq->cq_uk.cq_id | |
2686 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_CQ) | |
2687 | FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info->cq_resize) | |
2688 | FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info->pbl_chunk_size) | |
2689 | FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info->check_overflow) | |
2690 | FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info->virtual_map) | |
2691 | FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | |
2692 | FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | |
2693 | FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, |
2694 | cq->cq_uk.avoid_mem_cflct) | |
2695 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
2696 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
2697 | |
2698 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
2699 | |
2700 | print_hex_dump_debug("WQE: CQ_MODIFY WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
2701 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
2702 | if (post_sq) |
2703 | irdma_sc_cqp_post_sq(cqp); |
2704 | |
2705 | return 0; |
2706 | } |
2707 | |
2708 | /** |
2709 | * irdma_check_cqp_progress - check cqp processing progress |
2710 | * @timeout: timeout info struct |
2711 | * @dev: sc device struct |
2712 | */ |
2713 | void irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, struct irdma_sc_dev *dev) |
2714 | { |
2715 | u64 completed_ops = atomic64_read(v: &dev->cqp->completed_ops); |
2716 | |
2717 | if (timeout->compl_cqp_cmds != completed_ops) { |
2718 | timeout->compl_cqp_cmds = completed_ops; |
2719 | timeout->count = 0; |
2720 | } else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) { |
2721 | timeout->count++; |
2722 | } |
2723 | } |
2724 | |
2725 | /** |
2726 | * irdma_get_cqp_reg_info - get head and tail for cqp using registers |
2727 | * @cqp: struct for cqp hw |
2728 | * @val: cqp tail register value |
2729 | * @tail: wqtail register value |
2730 | * @error: cqp processing err |
2731 | */ |
2732 | static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val, |
2733 | u32 *tail, u32 *error) |
2734 | { |
2735 | *val = readl(addr: cqp->dev->hw_regs[IRDMA_CQPTAIL]); |
2736 | *tail = FIELD_GET(IRDMA_CQPTAIL_WQTAIL, *val); |
2737 | *error = FIELD_GET(IRDMA_CQPTAIL_CQP_OP_ERR, *val); |
2738 | } |
2739 | |
2740 | /** |
2741 | * irdma_cqp_poll_registers - poll cqp registers |
2742 | * @cqp: struct for cqp hw |
2743 | * @tail: wqtail register value |
2744 | * @count: how many times to try for completion |
2745 | */ |
2746 | static int irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, u32 tail, |
2747 | u32 count) |
2748 | { |
2749 | u32 i = 0; |
2750 | u32 newtail, error, val; |
2751 | |
2752 | while (i++ < count) { |
2753 | irdma_get_cqp_reg_info(cqp, val: &val, tail: &newtail, error: &error); |
2754 | if (error) { |
2755 | error = readl(addr: cqp->dev->hw_regs[IRDMA_CQPERRCODES]); |
2756 | ibdev_dbg(to_ibdev(cqp->dev), |
2757 | "CQP: CQPERRCODES error_code[x%08X]\n" , |
2758 | error); |
2759 | return -EIO; |
2760 | } |
2761 | if (newtail != tail) { |
2762 | /* SUCCESS */ |
2763 | IRDMA_RING_MOVE_TAIL(cqp->sq_ring); |
2764 | atomic64_inc(v: &cqp->completed_ops); |
2765 | return 0; |
2766 | } |
2767 | udelay(cqp->dev->hw_attrs.max_sleep_count); |
2768 | } |
2769 | |
2770 | return -ETIMEDOUT; |
2771 | } |
2772 | |
2773 | /** |
2774 | * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base |
2775 | * @dev: sc device struct |
2776 | * @buf: pointer to commit buffer |
2777 | * @buf_idx: buffer index |
2778 | * @obj_info: object info pointer |
2779 | * @rsrc_idx: indexs of memory resource |
2780 | */ |
2781 | static u64 irdma_sc_decode_fpm_commit(struct irdma_sc_dev *dev, __le64 *buf, |
2782 | u32 buf_idx, struct irdma_hmc_obj_info *obj_info, |
2783 | u32 rsrc_idx) |
2784 | { |
2785 | u64 temp; |
2786 | |
2787 | get_64bit_val(wqe_words: buf, byte_index: buf_idx, val: &temp); |
2788 | |
2789 | switch (rsrc_idx) { |
2790 | case IRDMA_HMC_IW_QP: |
2791 | obj_info[rsrc_idx].cnt = (u32)FIELD_GET(IRDMA_COMMIT_FPM_QPCNT, temp); |
2792 | break; |
2793 | case IRDMA_HMC_IW_CQ: |
2794 | obj_info[rsrc_idx].cnt = (u32)FLD_RS_64(dev, temp, IRDMA_COMMIT_FPM_CQCNT); |
2795 | break; |
2796 | case IRDMA_HMC_IW_APBVT_ENTRY: |
2797 | obj_info[rsrc_idx].cnt = 1; |
2798 | break; |
2799 | default: |
2800 | obj_info[rsrc_idx].cnt = (u32)temp; |
2801 | break; |
2802 | } |
2803 | |
2804 | obj_info[rsrc_idx].base = (temp >> IRDMA_COMMIT_FPM_BASE_S) * 512; |
2805 | |
2806 | return temp; |
2807 | } |
2808 | |
2809 | /** |
2810 | * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer |
2811 | * @dev: pointer to dev struct |
2812 | * @buf: ptr to fpm commit buffer |
2813 | * @info: ptr to irdma_hmc_obj_info struct |
2814 | * @sd: number of SDs for HMC objects |
2815 | * |
2816 | * parses fpm commit info and copy base value |
2817 | * of hmc objects in hmc_info |
2818 | */ |
2819 | static void |
2820 | irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 *buf, |
2821 | struct irdma_hmc_obj_info *info, u32 *sd) |
2822 | { |
2823 | u64 size; |
2824 | u32 i; |
2825 | u64 max_base = 0; |
2826 | u32 last_hmc_obj = 0; |
2827 | |
2828 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 0, obj_info: info, |
2829 | rsrc_idx: IRDMA_HMC_IW_QP); |
2830 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 8, obj_info: info, |
2831 | rsrc_idx: IRDMA_HMC_IW_CQ); |
2832 | /* skiping RSRVD */ |
2833 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 24, obj_info: info, |
2834 | rsrc_idx: IRDMA_HMC_IW_HTE); |
2835 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 32, obj_info: info, |
2836 | rsrc_idx: IRDMA_HMC_IW_ARP); |
2837 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 40, obj_info: info, |
2838 | rsrc_idx: IRDMA_HMC_IW_APBVT_ENTRY); |
2839 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 48, obj_info: info, |
2840 | rsrc_idx: IRDMA_HMC_IW_MR); |
2841 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 56, obj_info: info, |
2842 | rsrc_idx: IRDMA_HMC_IW_XF); |
2843 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 64, obj_info: info, |
2844 | rsrc_idx: IRDMA_HMC_IW_XFFL); |
2845 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 72, obj_info: info, |
2846 | rsrc_idx: IRDMA_HMC_IW_Q1); |
2847 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 80, obj_info: info, |
2848 | rsrc_idx: IRDMA_HMC_IW_Q1FL); |
2849 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 88, obj_info: info, |
2850 | rsrc_idx: IRDMA_HMC_IW_TIMER); |
2851 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 112, obj_info: info, |
2852 | rsrc_idx: IRDMA_HMC_IW_PBLE); |
2853 | /* skipping RSVD. */ |
2854 | if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) { |
2855 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 96, obj_info: info, |
2856 | rsrc_idx: IRDMA_HMC_IW_FSIMC); |
2857 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 104, obj_info: info, |
2858 | rsrc_idx: IRDMA_HMC_IW_FSIAV); |
2859 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 128, obj_info: info, |
2860 | rsrc_idx: IRDMA_HMC_IW_RRF); |
2861 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 136, obj_info: info, |
2862 | rsrc_idx: IRDMA_HMC_IW_RRFFL); |
2863 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 144, obj_info: info, |
2864 | rsrc_idx: IRDMA_HMC_IW_HDR); |
2865 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 152, obj_info: info, |
2866 | rsrc_idx: IRDMA_HMC_IW_MD); |
2867 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 160, obj_info: info, |
2868 | rsrc_idx: IRDMA_HMC_IW_OOISC); |
2869 | irdma_sc_decode_fpm_commit(dev, buf, buf_idx: 168, obj_info: info, |
2870 | rsrc_idx: IRDMA_HMC_IW_OOISCFFL); |
2871 | } |
2872 | |
2873 | /* searching for the last object in HMC to find the size of the HMC area. */ |
2874 | for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) { |
2875 | if (info[i].base > max_base) { |
2876 | max_base = info[i].base; |
2877 | last_hmc_obj = i; |
2878 | } |
2879 | } |
2880 | |
2881 | size = info[last_hmc_obj].cnt * info[last_hmc_obj].size + |
2882 | info[last_hmc_obj].base; |
2883 | |
2884 | if (size & 0x1FFFFF) |
2885 | *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */ |
2886 | else |
2887 | *sd = (u32)(size >> 21); |
2888 | |
2889 | } |
2890 | |
2891 | /** |
2892 | * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size |
2893 | * @buf: ptr to fpm query buffer |
2894 | * @buf_idx: index into buf |
2895 | * @obj_info: ptr to irdma_hmc_obj_info struct |
2896 | * @rsrc_idx: resource index into info |
2897 | * |
2898 | * Decode a 64 bit value from fpm query buffer into max count and size |
2899 | */ |
2900 | static u64 irdma_sc_decode_fpm_query(__le64 *buf, u32 buf_idx, |
2901 | struct irdma_hmc_obj_info *obj_info, |
2902 | u32 rsrc_idx) |
2903 | { |
2904 | u64 temp; |
2905 | u32 size; |
2906 | |
2907 | get_64bit_val(wqe_words: buf, byte_index: buf_idx, val: &temp); |
2908 | obj_info[rsrc_idx].max_cnt = (u32)temp; |
2909 | size = (u32)(temp >> 32); |
2910 | obj_info[rsrc_idx].size = BIT_ULL(size); |
2911 | |
2912 | return temp; |
2913 | } |
2914 | |
2915 | /** |
2916 | * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer |
2917 | * @dev: ptr to shared code device |
2918 | * @buf: ptr to fpm query buffer |
2919 | * @hmc_info: ptr to irdma_hmc_obj_info struct |
2920 | * @hmc_fpm_misc: ptr to fpm data |
2921 | * |
2922 | * parses fpm query buffer and copy max_cnt and |
2923 | * size value of hmc objects in hmc_info |
2924 | */ |
2925 | static int irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 *buf, |
2926 | struct irdma_hmc_info *hmc_info, |
2927 | struct irdma_hmc_fpm_misc *hmc_fpm_misc) |
2928 | { |
2929 | struct irdma_hmc_obj_info *obj_info; |
2930 | u64 temp; |
2931 | u32 size; |
2932 | u16 max_pe_sds; |
2933 | |
2934 | obj_info = hmc_info->hmc_obj; |
2935 | |
2936 | get_64bit_val(wqe_words: buf, byte_index: 0, val: &temp); |
2937 | hmc_info->first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp); |
2938 | max_pe_sds = (u16)FIELD_GET(IRDMA_QUERY_FPM_MAX_PE_SDS, temp); |
2939 | |
2940 | hmc_fpm_misc->max_sds = max_pe_sds; |
2941 | hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index; |
2942 | get_64bit_val(wqe_words: buf, byte_index: 8, val: &temp); |
2943 | obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_QPS, temp); |
2944 | size = (u32)(temp >> 32); |
2945 | obj_info[IRDMA_HMC_IW_QP].size = BIT_ULL(size); |
2946 | |
2947 | get_64bit_val(wqe_words: buf, byte_index: 16, val: &temp); |
2948 | obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_CQS, temp); |
2949 | size = (u32)(temp >> 32); |
2950 | obj_info[IRDMA_HMC_IW_CQ].size = BIT_ULL(size); |
2951 | |
2952 | irdma_sc_decode_fpm_query(buf, buf_idx: 32, obj_info, rsrc_idx: IRDMA_HMC_IW_HTE); |
2953 | irdma_sc_decode_fpm_query(buf, buf_idx: 40, obj_info, rsrc_idx: IRDMA_HMC_IW_ARP); |
2954 | |
2955 | obj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 8192; |
2956 | obj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 1; |
2957 | |
2958 | irdma_sc_decode_fpm_query(buf, buf_idx: 48, obj_info, rsrc_idx: IRDMA_HMC_IW_MR); |
2959 | irdma_sc_decode_fpm_query(buf, buf_idx: 56, obj_info, rsrc_idx: IRDMA_HMC_IW_XF); |
2960 | |
2961 | get_64bit_val(wqe_words: buf, byte_index: 64, val: &temp); |
2962 | obj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp; |
2963 | obj_info[IRDMA_HMC_IW_XFFL].size = 4; |
2964 | hmc_fpm_misc->xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp); |
2965 | if (!hmc_fpm_misc->xf_block_size) |
2966 | return -EINVAL; |
2967 | |
2968 | irdma_sc_decode_fpm_query(buf, buf_idx: 72, obj_info, rsrc_idx: IRDMA_HMC_IW_Q1); |
2969 | get_64bit_val(wqe_words: buf, byte_index: 80, val: &temp); |
2970 | obj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp; |
2971 | obj_info[IRDMA_HMC_IW_Q1FL].size = 4; |
2972 | |
2973 | hmc_fpm_misc->q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp); |
2974 | if (!hmc_fpm_misc->q1_block_size) |
2975 | return -EINVAL; |
2976 | |
2977 | irdma_sc_decode_fpm_query(buf, buf_idx: 88, obj_info, rsrc_idx: IRDMA_HMC_IW_TIMER); |
2978 | |
2979 | get_64bit_val(wqe_words: buf, byte_index: 112, val: &temp); |
2980 | obj_info[IRDMA_HMC_IW_PBLE].max_cnt = (u32)temp; |
2981 | obj_info[IRDMA_HMC_IW_PBLE].size = 8; |
2982 | |
2983 | get_64bit_val(wqe_words: buf, byte_index: 120, val: &temp); |
2984 | hmc_fpm_misc->max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp); |
2985 | hmc_fpm_misc->ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp); |
2986 | hmc_fpm_misc->timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp); |
2987 | if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) |
2988 | return 0; |
2989 | irdma_sc_decode_fpm_query(buf, buf_idx: 96, obj_info, rsrc_idx: IRDMA_HMC_IW_FSIMC); |
2990 | irdma_sc_decode_fpm_query(buf, buf_idx: 104, obj_info, rsrc_idx: IRDMA_HMC_IW_FSIAV); |
2991 | irdma_sc_decode_fpm_query(buf, buf_idx: 128, obj_info, rsrc_idx: IRDMA_HMC_IW_RRF); |
2992 | |
2993 | get_64bit_val(wqe_words: buf, byte_index: 136, val: &temp); |
2994 | obj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp; |
2995 | obj_info[IRDMA_HMC_IW_RRFFL].size = 4; |
2996 | hmc_fpm_misc->rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp); |
2997 | if (!hmc_fpm_misc->rrf_block_size && |
2998 | obj_info[IRDMA_HMC_IW_RRFFL].max_cnt) |
2999 | return -EINVAL; |
3000 | |
3001 | irdma_sc_decode_fpm_query(buf, buf_idx: 144, obj_info, rsrc_idx: IRDMA_HMC_IW_HDR); |
3002 | irdma_sc_decode_fpm_query(buf, buf_idx: 152, obj_info, rsrc_idx: IRDMA_HMC_IW_MD); |
3003 | irdma_sc_decode_fpm_query(buf, buf_idx: 160, obj_info, rsrc_idx: IRDMA_HMC_IW_OOISC); |
3004 | |
3005 | get_64bit_val(wqe_words: buf, byte_index: 168, val: &temp); |
3006 | obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp; |
3007 | obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4; |
3008 | hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp); |
3009 | if (!hmc_fpm_misc->ooiscf_block_size && |
3010 | obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt) |
3011 | return -EINVAL; |
3012 | |
3013 | return 0; |
3014 | } |
3015 | |
3016 | /** |
3017 | * irdma_sc_find_reg_cq - find cq ctx index |
3018 | * @ceq: ceq sc structure |
3019 | * @cq: cq sc structure |
3020 | */ |
3021 | static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq, |
3022 | struct irdma_sc_cq *cq) |
3023 | { |
3024 | u32 i; |
3025 | |
3026 | for (i = 0; i < ceq->reg_cq_size; i++) { |
3027 | if (cq == ceq->reg_cq[i]) |
3028 | return i; |
3029 | } |
3030 | |
3031 | return IRDMA_INVALID_CQ_IDX; |
3032 | } |
3033 | |
3034 | /** |
3035 | * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq |
3036 | * @ceq: ceq sc structure |
3037 | * @cq: cq sc structure |
3038 | */ |
3039 | int irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq) |
3040 | { |
3041 | unsigned long flags; |
3042 | |
3043 | spin_lock_irqsave(&ceq->req_cq_lock, flags); |
3044 | |
3045 | if (ceq->reg_cq_size == ceq->elem_cnt) { |
3046 | spin_unlock_irqrestore(lock: &ceq->req_cq_lock, flags); |
3047 | return -ENOMEM; |
3048 | } |
3049 | |
3050 | ceq->reg_cq[ceq->reg_cq_size++] = cq; |
3051 | |
3052 | spin_unlock_irqrestore(lock: &ceq->req_cq_lock, flags); |
3053 | |
3054 | return 0; |
3055 | } |
3056 | |
3057 | /** |
3058 | * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq |
3059 | * @ceq: ceq sc structure |
3060 | * @cq: cq sc structure |
3061 | */ |
3062 | void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq) |
3063 | { |
3064 | unsigned long flags; |
3065 | u32 cq_ctx_idx; |
3066 | |
3067 | spin_lock_irqsave(&ceq->req_cq_lock, flags); |
3068 | cq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq); |
3069 | if (cq_ctx_idx == IRDMA_INVALID_CQ_IDX) |
3070 | goto exit; |
3071 | |
3072 | ceq->reg_cq_size--; |
3073 | if (cq_ctx_idx != ceq->reg_cq_size) |
3074 | ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size]; |
3075 | ceq->reg_cq[ceq->reg_cq_size] = NULL; |
3076 | |
3077 | exit: |
3078 | spin_unlock_irqrestore(lock: &ceq->req_cq_lock, flags); |
3079 | } |
3080 | |
3081 | /** |
3082 | * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair |
3083 | * @cqp: IWARP control queue pair pointer |
3084 | * @info: IWARP control queue pair init info pointer |
3085 | * |
3086 | * Initializes the object and context buffers for a control Queue Pair. |
3087 | */ |
3088 | int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, |
3089 | struct irdma_cqp_init_info *info) |
3090 | { |
3091 | u8 hw_sq_size; |
3092 | |
3093 | if (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 || |
3094 | info->sq_size < IRDMA_CQP_SW_SQSIZE_4 || |
3095 | ((info->sq_size & (info->sq_size - 1)))) |
3096 | return -EINVAL; |
3097 | |
3098 | hw_sq_size = irdma_get_encoded_wqe_size(wqsize: info->sq_size, |
3099 | queue_type: IRDMA_QUEUE_TYPE_CQP); |
3100 | cqp->size = sizeof(*cqp); |
3101 | cqp->sq_size = info->sq_size; |
3102 | cqp->hw_sq_size = hw_sq_size; |
3103 | cqp->sq_base = info->sq; |
3104 | cqp->host_ctx = info->host_ctx; |
3105 | cqp->sq_pa = info->sq_pa; |
3106 | cqp->host_ctx_pa = info->host_ctx_pa; |
3107 | cqp->dev = info->dev; |
3108 | cqp->struct_ver = info->struct_ver; |
3109 | cqp->hw_maj_ver = info->hw_maj_ver; |
3110 | cqp->hw_min_ver = info->hw_min_ver; |
3111 | cqp->scratch_array = info->scratch_array; |
3112 | cqp->polarity = 0; |
3113 | cqp->en_datacenter_tcp = info->en_datacenter_tcp; |
3114 | cqp->ena_vf_count = info->ena_vf_count; |
3115 | cqp->hmc_profile = info->hmc_profile; |
3116 | cqp->ceqs_per_vf = info->ceqs_per_vf; |
3117 | cqp->disable_packed = info->disable_packed; |
3118 | cqp->rocev2_rto_policy = info->rocev2_rto_policy; |
3119 | cqp->protocol_used = info->protocol_used; |
3120 | memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params)); |
3121 | info->dev->cqp = cqp; |
3122 | |
3123 | IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size); |
3124 | cqp->requested_ops = 0; |
3125 | atomic64_set(v: &cqp->completed_ops, i: 0); |
3126 | /* for the cqp commands backlog. */ |
3127 | INIT_LIST_HEAD(list: &cqp->dev->cqp_cmd_head); |
3128 | |
3129 | writel(val: 0, addr: cqp->dev->hw_regs[IRDMA_CQPTAIL]); |
3130 | writel(val: 0, addr: cqp->dev->hw_regs[IRDMA_CQPDB]); |
3131 | writel(val: 0, addr: cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); |
3132 | |
3133 | ibdev_dbg(to_ibdev(cqp->dev), |
3134 | "WQE: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%pK] cqp[%p] polarity[x%04x]\n" , |
3135 | cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, |
3136 | (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity); |
3137 | return 0; |
3138 | } |
3139 | |
3140 | /** |
3141 | * irdma_sc_cqp_create - create cqp during bringup |
3142 | * @cqp: struct for cqp hw |
3143 | * @maj_err: If error, major err number |
3144 | * @min_err: If error, minor err number |
3145 | */ |
3146 | int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err) |
3147 | { |
3148 | u64 temp; |
3149 | u8 hw_rev; |
3150 | u32 cnt = 0, p1, p2, val = 0, err_code; |
3151 | int ret_code; |
3152 | |
3153 | hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev; |
3154 | cqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size, |
3155 | IRDMA_SD_BUF_ALIGNMENT); |
3156 | cqp->sdbuf.va = dma_alloc_coherent(dev: cqp->dev->hw->device, |
3157 | size: cqp->sdbuf.size, dma_handle: &cqp->sdbuf.pa, |
3158 | GFP_KERNEL); |
3159 | if (!cqp->sdbuf.va) |
3160 | return -ENOMEM; |
3161 | |
3162 | spin_lock_init(&cqp->dev->cqp_lock); |
3163 | |
3164 | temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) | |
3165 | FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) | |
3166 | FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) | |
3167 | FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf); |
3168 | if (hw_rev >= IRDMA_GEN_2) { |
3169 | temp |= FIELD_PREP(IRDMA_CQPHC_ROCEV2_RTO_POLICY, |
3170 | cqp->rocev2_rto_policy) | |
3171 | FIELD_PREP(IRDMA_CQPHC_PROTOCOL_USED, |
3172 | cqp->protocol_used); |
3173 | } |
3174 | |
3175 | set_64bit_val(wqe_words: cqp->host_ctx, byte_index: 0, val: temp); |
3176 | set_64bit_val(wqe_words: cqp->host_ctx, byte_index: 8, val: cqp->sq_pa); |
3177 | |
3178 | temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) | |
3179 | FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile); |
3180 | set_64bit_val(wqe_words: cqp->host_ctx, byte_index: 16, val: temp); |
3181 | set_64bit_val(wqe_words: cqp->host_ctx, byte_index: 24, val: (uintptr_t)cqp); |
3182 | temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) | |
3183 | FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver); |
3184 | if (hw_rev >= IRDMA_GEN_2) { |
3185 | temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) | |
3186 | FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor); |
3187 | } |
3188 | set_64bit_val(wqe_words: cqp->host_ctx, byte_index: 32, val: temp); |
3189 | set_64bit_val(wqe_words: cqp->host_ctx, byte_index: 40, val: 0); |
3190 | temp = 0; |
3191 | if (hw_rev >= IRDMA_GEN_2) { |
3192 | temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) | |
3193 | FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) | |
3194 | FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor); |
3195 | } |
3196 | set_64bit_val(wqe_words: cqp->host_ctx, byte_index: 48, val: temp); |
3197 | temp = 0; |
3198 | if (hw_rev >= IRDMA_GEN_2) { |
3199 | temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) | |
3200 | FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) | |
3201 | FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) | |
3202 | FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod); |
3203 | } |
3204 | set_64bit_val(wqe_words: cqp->host_ctx, byte_index: 56, val: temp); |
3205 | print_hex_dump_debug("WQE: CQP_HOST_CTX WQE" , DUMP_PREFIX_OFFSET, 16, |
3206 | 8, cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8, false); |
3207 | p1 = cqp->host_ctx_pa >> 32; |
3208 | p2 = (u32)cqp->host_ctx_pa; |
3209 | |
3210 | writel(val: p1, addr: cqp->dev->hw_regs[IRDMA_CCQPHIGH]); |
3211 | writel(val: p2, addr: cqp->dev->hw_regs[IRDMA_CCQPLOW]); |
3212 | |
3213 | do { |
3214 | if (cnt++ > cqp->dev->hw_attrs.max_done_count) { |
3215 | ret_code = -ETIMEDOUT; |
3216 | goto err; |
3217 | } |
3218 | udelay(cqp->dev->hw_attrs.max_sleep_count); |
3219 | val = readl(addr: cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); |
3220 | } while (!val); |
3221 | |
3222 | if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) { |
3223 | ret_code = -EOPNOTSUPP; |
3224 | goto err; |
3225 | } |
3226 | |
3227 | cqp->process_cqp_sds = irdma_update_sds_noccq; |
3228 | return 0; |
3229 | |
3230 | err: |
3231 | dma_free_coherent(dev: cqp->dev->hw->device, size: cqp->sdbuf.size, |
3232 | cpu_addr: cqp->sdbuf.va, dma_handle: cqp->sdbuf.pa); |
3233 | cqp->sdbuf.va = NULL; |
3234 | err_code = readl(addr: cqp->dev->hw_regs[IRDMA_CQPERRCODES]); |
3235 | *min_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MINOR_CODE, err_code); |
3236 | *maj_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MAJOR_CODE, err_code); |
3237 | return ret_code; |
3238 | } |
3239 | |
3240 | /** |
3241 | * irdma_sc_cqp_post_sq - post of cqp's sq |
3242 | * @cqp: struct for cqp hw |
3243 | */ |
3244 | void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp) |
3245 | { |
3246 | writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), addr: cqp->dev->cqp_db); |
3247 | |
3248 | ibdev_dbg(to_ibdev(cqp->dev), |
3249 | "WQE: CQP SQ head 0x%x tail 0x%x size 0x%x\n" , |
3250 | cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); |
3251 | } |
3252 | |
3253 | /** |
3254 | * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq |
3255 | * and pass back index |
3256 | * @cqp: CQP HW structure |
3257 | * @scratch: private data for CQP WQE |
3258 | * @wqe_idx: WQE index of CQP SQ |
3259 | */ |
3260 | __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch, |
3261 | u32 *wqe_idx) |
3262 | { |
3263 | __le64 *wqe = NULL; |
3264 | int ret_code; |
3265 | |
3266 | if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) { |
3267 | ibdev_dbg(to_ibdev(cqp->dev), |
3268 | "WQE: CQP SQ is full, head 0x%x tail 0x%x size 0x%x\n" , |
3269 | cqp->sq_ring.head, cqp->sq_ring.tail, |
3270 | cqp->sq_ring.size); |
3271 | return NULL; |
3272 | } |
3273 | IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code); |
3274 | if (ret_code) |
3275 | return NULL; |
3276 | |
3277 | cqp->requested_ops++; |
3278 | if (!*wqe_idx) |
3279 | cqp->polarity = !cqp->polarity; |
3280 | wqe = cqp->sq_base[*wqe_idx].elem; |
3281 | cqp->scratch_array[*wqe_idx] = scratch; |
3282 | IRDMA_CQP_INIT_WQE(wqe); |
3283 | |
3284 | return wqe; |
3285 | } |
3286 | |
3287 | /** |
3288 | * irdma_sc_cqp_destroy - destroy cqp during close |
3289 | * @cqp: struct for cqp hw |
3290 | */ |
3291 | int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp) |
3292 | { |
3293 | u32 cnt = 0, val; |
3294 | int ret_code = 0; |
3295 | |
3296 | writel(val: 0, addr: cqp->dev->hw_regs[IRDMA_CCQPHIGH]); |
3297 | writel(val: 0, addr: cqp->dev->hw_regs[IRDMA_CCQPLOW]); |
3298 | do { |
3299 | if (cnt++ > cqp->dev->hw_attrs.max_done_count) { |
3300 | ret_code = -ETIMEDOUT; |
3301 | break; |
3302 | } |
3303 | udelay(cqp->dev->hw_attrs.max_sleep_count); |
3304 | val = readl(addr: cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); |
3305 | } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE)); |
3306 | |
3307 | dma_free_coherent(dev: cqp->dev->hw->device, size: cqp->sdbuf.size, |
3308 | cpu_addr: cqp->sdbuf.va, dma_handle: cqp->sdbuf.pa); |
3309 | cqp->sdbuf.va = NULL; |
3310 | return ret_code; |
3311 | } |
3312 | |
3313 | /** |
3314 | * irdma_sc_ccq_arm - enable intr for control cq |
3315 | * @ccq: ccq sc struct |
3316 | */ |
3317 | void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq) |
3318 | { |
3319 | u64 temp_val; |
3320 | u16 sw_cq_sel; |
3321 | u8 arm_next_se; |
3322 | u8 arm_seq_num; |
3323 | |
3324 | get_64bit_val(wqe_words: ccq->cq_uk.shadow_area, byte_index: 32, val: &temp_val); |
3325 | sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); |
3326 | arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); |
3327 | arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); |
3328 | arm_seq_num++; |
3329 | temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) | |
3330 | FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) | |
3331 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | |
3332 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1); |
3333 | set_64bit_val(wqe_words: ccq->cq_uk.shadow_area, byte_index: 32, val: temp_val); |
3334 | |
3335 | dma_wmb(); /* make sure shadow area is updated before arming */ |
3336 | |
3337 | writel(val: ccq->cq_uk.cq_id, addr: ccq->dev->cq_arm_db); |
3338 | } |
3339 | |
3340 | /** |
3341 | * irdma_sc_ccq_get_cqe_info - get ccq's cq entry |
3342 | * @ccq: ccq sc struct |
3343 | * @info: completion q entry to return |
3344 | */ |
3345 | int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq, |
3346 | struct irdma_ccq_cqe_info *info) |
3347 | { |
3348 | u64 qp_ctx, temp, temp1; |
3349 | __le64 *cqe; |
3350 | struct irdma_sc_cqp *cqp; |
3351 | u32 wqe_idx; |
3352 | u32 error; |
3353 | u8 polarity; |
3354 | int ret_code = 0; |
3355 | |
3356 | if (ccq->cq_uk.avoid_mem_cflct) |
3357 | cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk); |
3358 | else |
3359 | cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk); |
3360 | |
3361 | get_64bit_val(wqe_words: cqe, byte_index: 24, val: &temp); |
3362 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, temp); |
3363 | if (polarity != ccq->cq_uk.polarity) |
3364 | return -ENOENT; |
3365 | |
3366 | /* Ensure CEQE contents are read after valid bit is checked */ |
3367 | dma_rmb(); |
3368 | |
3369 | get_64bit_val(wqe_words: cqe, byte_index: 8, val: &qp_ctx); |
3370 | cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx; |
3371 | info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp); |
3372 | info->maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR; |
3373 | info->min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp); |
3374 | if (info->error) { |
3375 | info->maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp); |
3376 | error = readl(addr: cqp->dev->hw_regs[IRDMA_CQPERRCODES]); |
3377 | ibdev_dbg(to_ibdev(cqp->dev), |
3378 | "CQP: CQPERRCODES error_code[x%08X]\n" , error); |
3379 | } |
3380 | |
3381 | wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, temp); |
3382 | info->scratch = cqp->scratch_array[wqe_idx]; |
3383 | |
3384 | get_64bit_val(wqe_words: cqe, byte_index: 16, val: &temp1); |
3385 | info->op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1); |
3386 | get_64bit_val(wqe_words: cqp->sq_base[wqe_idx].elem, byte_index: 24, val: &temp1); |
3387 | info->op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1); |
3388 | info->cqp = cqp; |
3389 | |
3390 | /* move the head for cq */ |
3391 | IRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code); |
3392 | if (!IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring)) |
3393 | ccq->cq_uk.polarity ^= 1; |
3394 | |
3395 | /* update cq tail in cq shadow memory also */ |
3396 | IRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring); |
3397 | set_64bit_val(wqe_words: ccq->cq_uk.shadow_area, byte_index: 0, |
3398 | IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring)); |
3399 | |
3400 | dma_wmb(); /* make sure shadow area is updated before moving tail */ |
3401 | |
3402 | IRDMA_RING_MOVE_TAIL(cqp->sq_ring); |
3403 | atomic64_inc(v: &cqp->completed_ops); |
3404 | |
3405 | return ret_code; |
3406 | } |
3407 | |
3408 | /** |
3409 | * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ |
3410 | * @cqp: struct for cqp hw |
3411 | * @op_code: cqp opcode for completion |
3412 | * @compl_info: completion q entry to return |
3413 | */ |
3414 | int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code, |
3415 | struct irdma_ccq_cqe_info *compl_info) |
3416 | { |
3417 | struct irdma_ccq_cqe_info info = {}; |
3418 | struct irdma_sc_cq *ccq; |
3419 | int ret_code = 0; |
3420 | u32 cnt = 0; |
3421 | |
3422 | ccq = cqp->dev->ccq; |
3423 | while (1) { |
3424 | if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count) |
3425 | return -ETIMEDOUT; |
3426 | |
3427 | if (irdma_sc_ccq_get_cqe_info(ccq, info: &info)) { |
3428 | udelay(cqp->dev->hw_attrs.max_sleep_count); |
3429 | continue; |
3430 | } |
3431 | if (info.error && info.op_code != IRDMA_CQP_OP_QUERY_STAG) { |
3432 | ret_code = -EIO; |
3433 | break; |
3434 | } |
3435 | /* make sure op code matches*/ |
3436 | if (op_code == info.op_code) |
3437 | break; |
3438 | ibdev_dbg(to_ibdev(cqp->dev), |
3439 | "WQE: opcode mismatch for my op code 0x%x, returned opcode %x\n" , |
3440 | op_code, info.op_code); |
3441 | } |
3442 | |
3443 | if (compl_info) |
3444 | memcpy(compl_info, &info, sizeof(*compl_info)); |
3445 | |
3446 | return ret_code; |
3447 | } |
3448 | |
3449 | /** |
3450 | * irdma_sc_manage_hmc_pm_func_table - manage of function table |
3451 | * @cqp: struct for cqp hw |
3452 | * @scratch: u64 saved to be used during cqp completion |
3453 | * @info: info for the manage function table operation |
3454 | * @post_sq: flag for cqp db to ring |
3455 | */ |
3456 | static int irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp, |
3457 | struct irdma_hmc_fcn_info *info, |
3458 | u64 scratch, bool post_sq) |
3459 | { |
3460 | __le64 *wqe; |
3461 | u64 hdr; |
3462 | |
3463 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
3464 | if (!wqe) |
3465 | return -ENOMEM; |
3466 | |
3467 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: 0); |
3468 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: 0); |
3469 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: 0); |
3470 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: 0); |
3471 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: 0); |
3472 | set_64bit_val(wqe_words: wqe, byte_index: 48, val: 0); |
3473 | set_64bit_val(wqe_words: wqe, byte_index: 56, val: 0); |
3474 | |
3475 | hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info->vf_id) | |
3476 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, |
3477 | IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE) | |
3478 | FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info->free_fcn) | |
3479 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
3480 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
3481 | |
3482 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
3483 | |
3484 | print_hex_dump_debug("WQE: MANAGE_HMC_PM_FUNC_TABLE WQE" , |
3485 | DUMP_PREFIX_OFFSET, 16, 8, wqe, |
3486 | IRDMA_CQP_WQE_SIZE * 8, false); |
3487 | if (post_sq) |
3488 | irdma_sc_cqp_post_sq(cqp); |
3489 | |
3490 | return 0; |
3491 | } |
3492 | |
3493 | /** |
3494 | * irdma_sc_commit_fpm_val_done - wait for cqp eqe completion |
3495 | * for fpm commit |
3496 | * @cqp: struct for cqp hw |
3497 | */ |
3498 | static int irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp) |
3499 | { |
3500 | return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL, |
3501 | NULL); |
3502 | } |
3503 | |
3504 | /** |
3505 | * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values |
3506 | * @cqp: struct for cqp hw |
3507 | * @scratch: u64 saved to be used during cqp completion |
3508 | * @hmc_fn_id: hmc function id |
3509 | * @commit_fpm_mem: Memory for fpm values |
3510 | * @post_sq: flag for cqp db to ring |
3511 | * @wait_type: poll ccq or cqp registers for cqp completion |
3512 | */ |
3513 | static int irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, |
3514 | u8 hmc_fn_id, |
3515 | struct irdma_dma_mem *commit_fpm_mem, |
3516 | bool post_sq, u8 wait_type) |
3517 | { |
3518 | __le64 *wqe; |
3519 | u64 hdr; |
3520 | u32 tail, val, error; |
3521 | int ret_code = 0; |
3522 | |
3523 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
3524 | if (!wqe) |
3525 | return -ENOMEM; |
3526 | |
3527 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: hmc_fn_id); |
3528 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: commit_fpm_mem->pa); |
3529 | |
3530 | hdr = FIELD_PREP(IRDMA_CQPSQ_BUFSIZE, IRDMA_COMMIT_FPM_BUF_SIZE) | |
3531 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_COMMIT_FPM_VAL) | |
3532 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
3533 | |
3534 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
3535 | |
3536 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
3537 | |
3538 | print_hex_dump_debug("WQE: COMMIT_FPM_VAL WQE" , DUMP_PREFIX_OFFSET, |
3539 | 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
3540 | irdma_get_cqp_reg_info(cqp, val: &val, tail: &tail, error: &error); |
3541 | |
3542 | if (post_sq) { |
3543 | irdma_sc_cqp_post_sq(cqp); |
3544 | if (wait_type == IRDMA_CQP_WAIT_POLL_REGS) |
3545 | ret_code = irdma_cqp_poll_registers(cqp, tail, |
3546 | count: cqp->dev->hw_attrs.max_done_count); |
3547 | else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ) |
3548 | ret_code = irdma_sc_commit_fpm_val_done(cqp); |
3549 | } |
3550 | |
3551 | return ret_code; |
3552 | } |
3553 | |
3554 | /** |
3555 | * irdma_sc_query_fpm_val_done - poll for cqp wqe completion for |
3556 | * query fpm |
3557 | * @cqp: struct for cqp hw |
3558 | */ |
3559 | static int irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp) |
3560 | { |
3561 | return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL, |
3562 | NULL); |
3563 | } |
3564 | |
3565 | /** |
3566 | * irdma_sc_query_fpm_val - cqp wqe query fpm values |
3567 | * @cqp: struct for cqp hw |
3568 | * @scratch: u64 saved to be used during cqp completion |
3569 | * @hmc_fn_id: hmc function id |
3570 | * @query_fpm_mem: memory for return fpm values |
3571 | * @post_sq: flag for cqp db to ring |
3572 | * @wait_type: poll ccq or cqp registers for cqp completion |
3573 | */ |
3574 | static int irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, |
3575 | u8 hmc_fn_id, |
3576 | struct irdma_dma_mem *query_fpm_mem, |
3577 | bool post_sq, u8 wait_type) |
3578 | { |
3579 | __le64 *wqe; |
3580 | u64 hdr; |
3581 | u32 tail, val, error; |
3582 | int ret_code = 0; |
3583 | |
3584 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
3585 | if (!wqe) |
3586 | return -ENOMEM; |
3587 | |
3588 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: hmc_fn_id); |
3589 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: query_fpm_mem->pa); |
3590 | |
3591 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_QUERY_FPM_VAL) | |
3592 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
3593 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
3594 | |
3595 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
3596 | |
3597 | print_hex_dump_debug("WQE: QUERY_FPM WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
3598 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
3599 | irdma_get_cqp_reg_info(cqp, val: &val, tail: &tail, error: &error); |
3600 | |
3601 | if (post_sq) { |
3602 | irdma_sc_cqp_post_sq(cqp); |
3603 | if (wait_type == IRDMA_CQP_WAIT_POLL_REGS) |
3604 | ret_code = irdma_cqp_poll_registers(cqp, tail, |
3605 | count: cqp->dev->hw_attrs.max_done_count); |
3606 | else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ) |
3607 | ret_code = irdma_sc_query_fpm_val_done(cqp); |
3608 | } |
3609 | |
3610 | return ret_code; |
3611 | } |
3612 | |
3613 | /** |
3614 | * irdma_sc_ceq_init - initialize ceq |
3615 | * @ceq: ceq sc structure |
3616 | * @info: ceq initialization info |
3617 | */ |
3618 | int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq, |
3619 | struct irdma_ceq_init_info *info) |
3620 | { |
3621 | u32 pble_obj_cnt; |
3622 | |
3623 | if (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size || |
3624 | info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size) |
3625 | return -EINVAL; |
3626 | |
3627 | if (info->ceq_id >= info->dev->hmc_fpm_misc.max_ceqs) |
3628 | return -EINVAL; |
3629 | pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; |
3630 | |
3631 | if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) |
3632 | return -EINVAL; |
3633 | |
3634 | ceq->size = sizeof(*ceq); |
3635 | ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base; |
3636 | ceq->ceq_id = info->ceq_id; |
3637 | ceq->dev = info->dev; |
3638 | ceq->elem_cnt = info->elem_cnt; |
3639 | ceq->ceq_elem_pa = info->ceqe_pa; |
3640 | ceq->virtual_map = info->virtual_map; |
3641 | ceq->itr_no_expire = info->itr_no_expire; |
3642 | ceq->reg_cq = info->reg_cq; |
3643 | ceq->reg_cq_size = 0; |
3644 | spin_lock_init(&ceq->req_cq_lock); |
3645 | ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0); |
3646 | ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0); |
3647 | ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL); |
3648 | ceq->tph_en = info->tph_en; |
3649 | ceq->tph_val = info->tph_val; |
3650 | ceq->vsi = info->vsi; |
3651 | ceq->polarity = 1; |
3652 | IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt); |
3653 | ceq->dev->ceq[info->ceq_id] = ceq; |
3654 | |
3655 | return 0; |
3656 | } |
3657 | |
3658 | /** |
3659 | * irdma_sc_ceq_create - create ceq wqe |
3660 | * @ceq: ceq sc structure |
3661 | * @scratch: u64 saved to be used during cqp completion |
3662 | * @post_sq: flag for cqp db to ring |
3663 | */ |
3664 | |
3665 | static int irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch, |
3666 | bool post_sq) |
3667 | { |
3668 | struct irdma_sc_cqp *cqp; |
3669 | __le64 *wqe; |
3670 | u64 hdr; |
3671 | |
3672 | cqp = ceq->dev->cqp; |
3673 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
3674 | if (!wqe) |
3675 | return -ENOMEM; |
3676 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: ceq->elem_cnt); |
3677 | set_64bit_val(wqe_words: wqe, byte_index: 32, |
3678 | val: (ceq->virtual_map ? 0 : ceq->ceq_elem_pa)); |
3679 | set_64bit_val(wqe_words: wqe, byte_index: 48, |
3680 | val: (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0)); |
3681 | set_64bit_val(wqe_words: wqe, byte_index: 56, |
3682 | FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) | |
3683 | FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx)); |
3684 | hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) | |
3685 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CEQ) | |
3686 | FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | |
3687 | FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | |
3688 | FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) | |
3689 | FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | |
3690 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
3691 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
3692 | |
3693 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
3694 | |
3695 | print_hex_dump_debug("WQE: CEQ_CREATE WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
3696 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
3697 | if (post_sq) |
3698 | irdma_sc_cqp_post_sq(cqp); |
3699 | |
3700 | return 0; |
3701 | } |
3702 | |
3703 | /** |
3704 | * irdma_sc_cceq_create_done - poll for control ceq wqe to complete |
3705 | * @ceq: ceq sc structure |
3706 | */ |
3707 | static int irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq) |
3708 | { |
3709 | struct irdma_sc_cqp *cqp; |
3710 | |
3711 | cqp = ceq->dev->cqp; |
3712 | return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ, |
3713 | NULL); |
3714 | } |
3715 | |
3716 | /** |
3717 | * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete |
3718 | * @ceq: ceq sc structure |
3719 | */ |
3720 | int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq) |
3721 | { |
3722 | struct irdma_sc_cqp *cqp; |
3723 | |
3724 | if (ceq->reg_cq) |
3725 | irdma_sc_remove_cq_ctx(ceq, cq: ceq->dev->ccq); |
3726 | |
3727 | cqp = ceq->dev->cqp; |
3728 | cqp->process_cqp_sds = irdma_update_sds_noccq; |
3729 | |
3730 | return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ, |
3731 | NULL); |
3732 | } |
3733 | |
3734 | /** |
3735 | * irdma_sc_cceq_create - create cceq |
3736 | * @ceq: ceq sc structure |
3737 | * @scratch: u64 saved to be used during cqp completion |
3738 | */ |
3739 | int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch) |
3740 | { |
3741 | int ret_code; |
3742 | struct irdma_sc_dev *dev = ceq->dev; |
3743 | |
3744 | dev->ccq->vsi = ceq->vsi; |
3745 | if (ceq->reg_cq) { |
3746 | ret_code = irdma_sc_add_cq_ctx(ceq, cq: ceq->dev->ccq); |
3747 | if (ret_code) |
3748 | return ret_code; |
3749 | } |
3750 | |
3751 | ret_code = irdma_sc_ceq_create(ceq, scratch, post_sq: true); |
3752 | if (!ret_code) |
3753 | return irdma_sc_cceq_create_done(ceq); |
3754 | |
3755 | return ret_code; |
3756 | } |
3757 | |
3758 | /** |
3759 | * irdma_sc_ceq_destroy - destroy ceq |
3760 | * @ceq: ceq sc structure |
3761 | * @scratch: u64 saved to be used during cqp completion |
3762 | * @post_sq: flag for cqp db to ring |
3763 | */ |
3764 | int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq) |
3765 | { |
3766 | struct irdma_sc_cqp *cqp; |
3767 | __le64 *wqe; |
3768 | u64 hdr; |
3769 | |
3770 | cqp = ceq->dev->cqp; |
3771 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
3772 | if (!wqe) |
3773 | return -ENOMEM; |
3774 | |
3775 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: ceq->elem_cnt); |
3776 | set_64bit_val(wqe_words: wqe, byte_index: 48, val: ceq->first_pm_pbl_idx); |
3777 | hdr = ceq->ceq_id | |
3778 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CEQ) | |
3779 | FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | |
3780 | FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | |
3781 | FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | |
3782 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
3783 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
3784 | |
3785 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
3786 | |
3787 | print_hex_dump_debug("WQE: CEQ_DESTROY WQE" , DUMP_PREFIX_OFFSET, 16, |
3788 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
3789 | if (post_sq) |
3790 | irdma_sc_cqp_post_sq(cqp); |
3791 | |
3792 | return 0; |
3793 | } |
3794 | |
3795 | /** |
3796 | * irdma_sc_process_ceq - process ceq |
3797 | * @dev: sc device struct |
3798 | * @ceq: ceq sc structure |
3799 | * |
3800 | * It is expected caller serializes this function with cleanup_ceqes() |
3801 | * because these functions manipulate the same ceq |
3802 | */ |
3803 | void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq) |
3804 | { |
3805 | u64 temp; |
3806 | __le64 *ceqe; |
3807 | struct irdma_sc_cq *cq = NULL; |
3808 | struct irdma_sc_cq *temp_cq; |
3809 | u8 polarity; |
3810 | u32 cq_idx; |
3811 | unsigned long flags; |
3812 | |
3813 | do { |
3814 | cq_idx = 0; |
3815 | ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq); |
3816 | get_64bit_val(wqe_words: ceqe, byte_index: 0, val: &temp); |
3817 | polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp); |
3818 | if (polarity != ceq->polarity) |
3819 | return NULL; |
3820 | |
3821 | temp_cq = (struct irdma_sc_cq *)(unsigned long)(temp << 1); |
3822 | if (!temp_cq) { |
3823 | cq_idx = IRDMA_INVALID_CQ_IDX; |
3824 | IRDMA_RING_MOVE_TAIL(ceq->ceq_ring); |
3825 | |
3826 | if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring)) |
3827 | ceq->polarity ^= 1; |
3828 | continue; |
3829 | } |
3830 | |
3831 | cq = temp_cq; |
3832 | if (ceq->reg_cq) { |
3833 | spin_lock_irqsave(&ceq->req_cq_lock, flags); |
3834 | cq_idx = irdma_sc_find_reg_cq(ceq, cq); |
3835 | spin_unlock_irqrestore(lock: &ceq->req_cq_lock, flags); |
3836 | } |
3837 | |
3838 | IRDMA_RING_MOVE_TAIL(ceq->ceq_ring); |
3839 | if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring)) |
3840 | ceq->polarity ^= 1; |
3841 | } while (cq_idx == IRDMA_INVALID_CQ_IDX); |
3842 | |
3843 | if (cq) |
3844 | irdma_sc_cq_ack(cq); |
3845 | return cq; |
3846 | } |
3847 | |
3848 | /** |
3849 | * irdma_sc_cleanup_ceqes - clear the valid ceqes ctx matching the cq |
3850 | * @cq: cq for which the ceqes need to be cleaned up |
3851 | * @ceq: ceq ptr |
3852 | * |
3853 | * The function is called after the cq is destroyed to cleanup |
3854 | * its pending ceqe entries. It is expected caller serializes this |
3855 | * function with process_ceq() in interrupt context. |
3856 | */ |
3857 | void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq) |
3858 | { |
3859 | struct irdma_sc_cq *next_cq; |
3860 | u8 ceq_polarity = ceq->polarity; |
3861 | __le64 *ceqe; |
3862 | u8 polarity; |
3863 | u64 temp; |
3864 | int next; |
3865 | u32 i; |
3866 | |
3867 | next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0); |
3868 | |
3869 | for (i = 1; i <= IRDMA_RING_SIZE(*ceq); i++) { |
3870 | ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next); |
3871 | |
3872 | get_64bit_val(wqe_words: ceqe, byte_index: 0, val: &temp); |
3873 | polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp); |
3874 | if (polarity != ceq_polarity) |
3875 | return; |
3876 | |
3877 | next_cq = (struct irdma_sc_cq *)(unsigned long)(temp << 1); |
3878 | if (cq == next_cq) |
3879 | set_64bit_val(wqe_words: ceqe, byte_index: 0, val: temp & IRDMA_CEQE_VALID); |
3880 | |
3881 | next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i); |
3882 | if (!next) |
3883 | ceq_polarity ^= 1; |
3884 | } |
3885 | } |
3886 | |
3887 | /** |
3888 | * irdma_sc_aeq_init - initialize aeq |
3889 | * @aeq: aeq structure ptr |
3890 | * @info: aeq initialization info |
3891 | */ |
3892 | int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq, |
3893 | struct irdma_aeq_init_info *info) |
3894 | { |
3895 | u32 pble_obj_cnt; |
3896 | |
3897 | if (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size || |
3898 | info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size) |
3899 | return -EINVAL; |
3900 | |
3901 | pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; |
3902 | |
3903 | if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) |
3904 | return -EINVAL; |
3905 | |
3906 | aeq->size = sizeof(*aeq); |
3907 | aeq->polarity = 1; |
3908 | aeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base; |
3909 | aeq->dev = info->dev; |
3910 | aeq->elem_cnt = info->elem_cnt; |
3911 | aeq->aeq_elem_pa = info->aeq_elem_pa; |
3912 | IRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt); |
3913 | aeq->virtual_map = info->virtual_map; |
3914 | aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL); |
3915 | aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0); |
3916 | aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0); |
3917 | aeq->msix_idx = info->msix_idx; |
3918 | info->dev->aeq = aeq; |
3919 | |
3920 | return 0; |
3921 | } |
3922 | |
3923 | /** |
3924 | * irdma_sc_aeq_create - create aeq |
3925 | * @aeq: aeq structure ptr |
3926 | * @scratch: u64 saved to be used during cqp completion |
3927 | * @post_sq: flag for cqp db to ring |
3928 | */ |
3929 | static int irdma_sc_aeq_create(struct irdma_sc_aeq *aeq, u64 scratch, |
3930 | bool post_sq) |
3931 | { |
3932 | __le64 *wqe; |
3933 | struct irdma_sc_cqp *cqp; |
3934 | u64 hdr; |
3935 | |
3936 | cqp = aeq->dev->cqp; |
3937 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
3938 | if (!wqe) |
3939 | return -ENOMEM; |
3940 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: aeq->elem_cnt); |
3941 | set_64bit_val(wqe_words: wqe, byte_index: 32, |
3942 | val: (aeq->virtual_map ? 0 : aeq->aeq_elem_pa)); |
3943 | set_64bit_val(wqe_words: wqe, byte_index: 48, |
3944 | val: (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0)); |
3945 | |
3946 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_AEQ) | |
3947 | FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | |
3948 | FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | |
3949 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
3950 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
3951 | |
3952 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
3953 | |
3954 | print_hex_dump_debug("WQE: AEQ_CREATE WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
3955 | wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
3956 | if (post_sq) |
3957 | irdma_sc_cqp_post_sq(cqp); |
3958 | |
3959 | return 0; |
3960 | } |
3961 | |
3962 | /** |
3963 | * irdma_sc_aeq_destroy - destroy aeq during close |
3964 | * @aeq: aeq structure ptr |
3965 | * @scratch: u64 saved to be used during cqp completion |
3966 | * @post_sq: flag for cqp db to ring |
3967 | */ |
3968 | static int irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch, |
3969 | bool post_sq) |
3970 | { |
3971 | __le64 *wqe; |
3972 | struct irdma_sc_cqp *cqp; |
3973 | struct irdma_sc_dev *dev; |
3974 | u64 hdr; |
3975 | |
3976 | dev = aeq->dev; |
3977 | writel(val: 0, addr: dev->hw_regs[IRDMA_PFINT_AEQCTL]); |
3978 | |
3979 | cqp = dev->cqp; |
3980 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
3981 | if (!wqe) |
3982 | return -ENOMEM; |
3983 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: aeq->elem_cnt); |
3984 | set_64bit_val(wqe_words: wqe, byte_index: 48, val: aeq->first_pm_pbl_idx); |
3985 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_AEQ) | |
3986 | FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | |
3987 | FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | |
3988 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
3989 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
3990 | |
3991 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
3992 | |
3993 | print_hex_dump_debug("WQE: AEQ_DESTROY WQE" , DUMP_PREFIX_OFFSET, 16, |
3994 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
3995 | if (post_sq) |
3996 | irdma_sc_cqp_post_sq(cqp); |
3997 | return 0; |
3998 | } |
3999 | |
4000 | /** |
4001 | * irdma_sc_get_next_aeqe - get next aeq entry |
4002 | * @aeq: aeq structure ptr |
4003 | * @info: aeqe info to be returned |
4004 | */ |
4005 | int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq, |
4006 | struct irdma_aeqe_info *info) |
4007 | { |
4008 | u64 temp, compl_ctx; |
4009 | __le64 *aeqe; |
4010 | u8 ae_src; |
4011 | u8 polarity; |
4012 | |
4013 | aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq); |
4014 | get_64bit_val(wqe_words: aeqe, byte_index: 8, val: &temp); |
4015 | polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp); |
4016 | |
4017 | if (aeq->polarity != polarity) |
4018 | return -ENOENT; |
4019 | |
4020 | /* Ensure AEQE contents are read after valid bit is checked */ |
4021 | dma_rmb(); |
4022 | |
4023 | get_64bit_val(wqe_words: aeqe, byte_index: 0, val: &compl_ctx); |
4024 | |
4025 | print_hex_dump_debug("WQE: AEQ_ENTRY WQE" , DUMP_PREFIX_OFFSET, 16, 8, |
4026 | aeqe, 16, false); |
4027 | |
4028 | ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp); |
4029 | info->wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp); |
4030 | info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) | |
4031 | ((u32)FIELD_GET(IRDMA_AEQE_QPCQID_HI, temp) << 18); |
4032 | info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp); |
4033 | info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp); |
4034 | info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp); |
4035 | info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp); |
4036 | info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp); |
4037 | |
4038 | info->ae_src = ae_src; |
4039 | switch (info->ae_id) { |
4040 | case IRDMA_AE_PRIV_OPERATION_DENIED: |
4041 | case IRDMA_AE_AMP_INVALIDATE_TYPE1_MW: |
4042 | case IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW: |
4043 | case IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG: |
4044 | case IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH: |
4045 | case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: |
4046 | case IRDMA_AE_UDA_XMIT_BAD_PD: |
4047 | case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT: |
4048 | case IRDMA_AE_BAD_CLOSE: |
4049 | case IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO: |
4050 | case IRDMA_AE_STAG_ZERO_INVALID: |
4051 | case IRDMA_AE_IB_RREQ_AND_Q1_FULL: |
4052 | case IRDMA_AE_IB_INVALID_REQUEST: |
4053 | case IRDMA_AE_WQE_UNEXPECTED_OPCODE: |
4054 | case IRDMA_AE_IB_REMOTE_ACCESS_ERROR: |
4055 | case IRDMA_AE_IB_REMOTE_OP_ERROR: |
4056 | case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION: |
4057 | case IRDMA_AE_DDP_UBE_INVALID_MO: |
4058 | case IRDMA_AE_DDP_UBE_INVALID_QN: |
4059 | case IRDMA_AE_DDP_NO_L_BIT: |
4060 | case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION: |
4061 | case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE: |
4062 | case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST: |
4063 | case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP: |
4064 | case IRDMA_AE_ROCE_RSP_LENGTH_ERROR: |
4065 | case IRDMA_AE_INVALID_ARP_ENTRY: |
4066 | case IRDMA_AE_INVALID_TCP_OPTION_RCVD: |
4067 | case IRDMA_AE_STALE_ARP_ENTRY: |
4068 | case IRDMA_AE_INVALID_AH_ENTRY: |
4069 | case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: |
4070 | case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: |
4071 | case IRDMA_AE_LLP_TOO_MANY_RETRIES: |
4072 | case IRDMA_AE_LLP_DOUBT_REACHABILITY: |
4073 | case IRDMA_AE_LLP_CONNECTION_ESTABLISHED: |
4074 | case IRDMA_AE_RESET_SENT: |
4075 | case IRDMA_AE_TERMINATE_SENT: |
4076 | case IRDMA_AE_RESET_NOT_SENT: |
4077 | case IRDMA_AE_LCE_QP_CATASTROPHIC: |
4078 | case IRDMA_AE_QP_SUSPEND_COMPLETE: |
4079 | case IRDMA_AE_UDA_L4LEN_INVALID: |
4080 | info->qp = true; |
4081 | info->compl_ctx = compl_ctx; |
4082 | break; |
4083 | case IRDMA_AE_LCE_CQ_CATASTROPHIC: |
4084 | info->cq = true; |
4085 | info->compl_ctx = compl_ctx << 1; |
4086 | ae_src = IRDMA_AE_SOURCE_RSVD; |
4087 | break; |
4088 | case IRDMA_AE_ROCE_EMPTY_MCG: |
4089 | case IRDMA_AE_ROCE_BAD_MC_IP_ADDR: |
4090 | case IRDMA_AE_ROCE_BAD_MC_QPID: |
4091 | case IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH: |
4092 | fallthrough; |
4093 | case IRDMA_AE_LLP_CONNECTION_RESET: |
4094 | case IRDMA_AE_LLP_SYN_RECEIVED: |
4095 | case IRDMA_AE_LLP_FIN_RECEIVED: |
4096 | case IRDMA_AE_LLP_CLOSE_COMPLETE: |
4097 | case IRDMA_AE_LLP_TERMINATE_RECEIVED: |
4098 | case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE: |
4099 | ae_src = IRDMA_AE_SOURCE_RSVD; |
4100 | info->qp = true; |
4101 | info->compl_ctx = compl_ctx; |
4102 | break; |
4103 | default: |
4104 | break; |
4105 | } |
4106 | |
4107 | switch (ae_src) { |
4108 | case IRDMA_AE_SOURCE_RQ: |
4109 | case IRDMA_AE_SOURCE_RQ_0011: |
4110 | info->qp = true; |
4111 | info->rq = true; |
4112 | info->compl_ctx = compl_ctx; |
4113 | break; |
4114 | case IRDMA_AE_SOURCE_CQ: |
4115 | case IRDMA_AE_SOURCE_CQ_0110: |
4116 | case IRDMA_AE_SOURCE_CQ_1010: |
4117 | case IRDMA_AE_SOURCE_CQ_1110: |
4118 | info->cq = true; |
4119 | info->compl_ctx = compl_ctx << 1; |
4120 | break; |
4121 | case IRDMA_AE_SOURCE_SQ: |
4122 | case IRDMA_AE_SOURCE_SQ_0111: |
4123 | info->qp = true; |
4124 | info->sq = true; |
4125 | info->compl_ctx = compl_ctx; |
4126 | break; |
4127 | case IRDMA_AE_SOURCE_IN_RR_WR: |
4128 | case IRDMA_AE_SOURCE_IN_RR_WR_1011: |
4129 | info->qp = true; |
4130 | info->compl_ctx = compl_ctx; |
4131 | info->in_rdrsp_wr = true; |
4132 | break; |
4133 | case IRDMA_AE_SOURCE_OUT_RR: |
4134 | case IRDMA_AE_SOURCE_OUT_RR_1111: |
4135 | info->qp = true; |
4136 | info->compl_ctx = compl_ctx; |
4137 | info->out_rdrsp = true; |
4138 | break; |
4139 | case IRDMA_AE_SOURCE_RSVD: |
4140 | default: |
4141 | break; |
4142 | } |
4143 | |
4144 | IRDMA_RING_MOVE_TAIL(aeq->aeq_ring); |
4145 | if (!IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring)) |
4146 | aeq->polarity ^= 1; |
4147 | |
4148 | return 0; |
4149 | } |
4150 | |
4151 | /** |
4152 | * irdma_sc_repost_aeq_entries - repost completed aeq entries |
4153 | * @dev: sc device struct |
4154 | * @count: allocate count |
4155 | */ |
4156 | void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count) |
4157 | { |
4158 | writel(val: count, addr: dev->hw_regs[IRDMA_AEQALLOC]); |
4159 | } |
4160 | |
4161 | /** |
4162 | * irdma_sc_ccq_init - initialize control cq |
4163 | * @cq: sc's cq ctruct |
4164 | * @info: info for control cq initialization |
4165 | */ |
4166 | int irdma_sc_ccq_init(struct irdma_sc_cq *cq, struct irdma_ccq_init_info *info) |
4167 | { |
4168 | u32 pble_obj_cnt; |
4169 | |
4170 | if (info->num_elem < info->dev->hw_attrs.uk_attrs.min_hw_cq_size || |
4171 | info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size) |
4172 | return -EINVAL; |
4173 | |
4174 | if (info->ceq_id >= info->dev->hmc_fpm_misc.max_ceqs) |
4175 | return -EINVAL; |
4176 | |
4177 | pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; |
4178 | |
4179 | if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) |
4180 | return -EINVAL; |
4181 | |
4182 | cq->cq_pa = info->cq_pa; |
4183 | cq->cq_uk.cq_base = info->cq_base; |
4184 | cq->shadow_area_pa = info->shadow_area_pa; |
4185 | cq->cq_uk.shadow_area = info->shadow_area; |
4186 | cq->shadow_read_threshold = info->shadow_read_threshold; |
4187 | cq->dev = info->dev; |
4188 | cq->ceq_id = info->ceq_id; |
4189 | cq->cq_uk.cq_size = info->num_elem; |
4190 | cq->cq_type = IRDMA_CQ_TYPE_CQP; |
4191 | cq->ceqe_mask = info->ceqe_mask; |
4192 | IRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem); |
4193 | cq->cq_uk.cq_id = 0; /* control cq is id 0 always */ |
4194 | cq->ceq_id_valid = info->ceq_id_valid; |
4195 | cq->tph_en = info->tph_en; |
4196 | cq->tph_val = info->tph_val; |
4197 | cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct; |
4198 | cq->pbl_list = info->pbl_list; |
4199 | cq->virtual_map = info->virtual_map; |
4200 | cq->pbl_chunk_size = info->pbl_chunk_size; |
4201 | cq->first_pm_pbl_idx = info->first_pm_pbl_idx; |
4202 | cq->cq_uk.polarity = true; |
4203 | cq->vsi = info->vsi; |
4204 | cq->cq_uk.cq_ack_db = cq->dev->cq_ack_db; |
4205 | |
4206 | /* Only applicable to CQs other than CCQ so initialize to zero */ |
4207 | cq->cq_uk.cqe_alloc_db = NULL; |
4208 | |
4209 | info->dev->ccq = cq; |
4210 | return 0; |
4211 | } |
4212 | |
4213 | /** |
4214 | * irdma_sc_ccq_create_done - poll cqp for ccq create |
4215 | * @ccq: ccq sc struct |
4216 | */ |
4217 | static inline int irdma_sc_ccq_create_done(struct irdma_sc_cq *ccq) |
4218 | { |
4219 | struct irdma_sc_cqp *cqp; |
4220 | |
4221 | cqp = ccq->dev->cqp; |
4222 | |
4223 | return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL); |
4224 | } |
4225 | |
4226 | /** |
4227 | * irdma_sc_ccq_create - create control cq |
4228 | * @ccq: ccq sc struct |
4229 | * @scratch: u64 saved to be used during cqp completion |
4230 | * @check_overflow: overlow flag for ccq |
4231 | * @post_sq: flag for cqp db to ring |
4232 | */ |
4233 | int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch, |
4234 | bool check_overflow, bool post_sq) |
4235 | { |
4236 | int ret_code; |
4237 | |
4238 | ret_code = irdma_sc_cq_create(cq: ccq, scratch, check_overflow, post_sq); |
4239 | if (ret_code) |
4240 | return ret_code; |
4241 | |
4242 | if (post_sq) { |
4243 | ret_code = irdma_sc_ccq_create_done(ccq); |
4244 | if (ret_code) |
4245 | return ret_code; |
4246 | } |
4247 | ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd; |
4248 | |
4249 | return 0; |
4250 | } |
4251 | |
4252 | /** |
4253 | * irdma_sc_ccq_destroy - destroy ccq during close |
4254 | * @ccq: ccq sc struct |
4255 | * @scratch: u64 saved to be used during cqp completion |
4256 | * @post_sq: flag for cqp db to ring |
4257 | */ |
4258 | int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq) |
4259 | { |
4260 | struct irdma_sc_cqp *cqp; |
4261 | __le64 *wqe; |
4262 | u64 hdr; |
4263 | int ret_code = 0; |
4264 | u32 tail, val, error; |
4265 | |
4266 | cqp = ccq->dev->cqp; |
4267 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
4268 | if (!wqe) |
4269 | return -ENOMEM; |
4270 | |
4271 | set_64bit_val(wqe_words: wqe, byte_index: 0, val: ccq->cq_uk.cq_size); |
4272 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: (uintptr_t)ccq >> 1); |
4273 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: ccq->shadow_area_pa); |
4274 | |
4275 | hdr = ccq->cq_uk.cq_id | |
4276 | FLD_LS_64(ccq->dev, (ccq->ceq_id_valid ? ccq->ceq_id : 0), |
4277 | IRDMA_CQPSQ_CQ_CEQID) | |
4278 | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) | |
4279 | FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq->ceqe_mask) | |
4280 | FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq->ceq_id_valid) | |
4281 | FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq->tph_en) | |
4282 | FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq->cq_uk.avoid_mem_cflct) | |
4283 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
4284 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
4285 | |
4286 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
4287 | |
4288 | print_hex_dump_debug("WQE: CCQ_DESTROY WQE" , DUMP_PREFIX_OFFSET, 16, |
4289 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
4290 | irdma_get_cqp_reg_info(cqp, val: &val, tail: &tail, error: &error); |
4291 | |
4292 | if (post_sq) { |
4293 | irdma_sc_cqp_post_sq(cqp); |
4294 | ret_code = irdma_cqp_poll_registers(cqp, tail, |
4295 | count: cqp->dev->hw_attrs.max_done_count); |
4296 | } |
4297 | |
4298 | cqp->process_cqp_sds = irdma_update_sds_noccq; |
4299 | |
4300 | return ret_code; |
4301 | } |
4302 | |
4303 | /** |
4304 | * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info |
4305 | * @dev : ptr to irdma_dev struct |
4306 | * @hmc_fn_id: hmc function id |
4307 | */ |
4308 | int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id) |
4309 | { |
4310 | struct irdma_hmc_info *hmc_info; |
4311 | struct irdma_hmc_fpm_misc *hmc_fpm_misc; |
4312 | struct irdma_dma_mem query_fpm_mem; |
4313 | int ret_code = 0; |
4314 | u8 wait_type; |
4315 | |
4316 | hmc_info = dev->hmc_info; |
4317 | hmc_fpm_misc = &dev->hmc_fpm_misc; |
4318 | query_fpm_mem.pa = dev->fpm_query_buf_pa; |
4319 | query_fpm_mem.va = dev->fpm_query_buf; |
4320 | hmc_info->hmc_fn_id = hmc_fn_id; |
4321 | wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS; |
4322 | |
4323 | ret_code = irdma_sc_query_fpm_val(cqp: dev->cqp, scratch: 0, hmc_fn_id: hmc_info->hmc_fn_id, |
4324 | query_fpm_mem: &query_fpm_mem, post_sq: true, wait_type); |
4325 | if (ret_code) |
4326 | return ret_code; |
4327 | |
4328 | /* parse the fpm_query_buf and fill hmc obj info */ |
4329 | ret_code = irdma_sc_parse_fpm_query_buf(dev, buf: query_fpm_mem.va, hmc_info, |
4330 | hmc_fpm_misc); |
4331 | |
4332 | print_hex_dump_debug("HMC: QUERY FPM BUFFER" , DUMP_PREFIX_OFFSET, 16, |
4333 | 8, query_fpm_mem.va, IRDMA_QUERY_FPM_BUF_SIZE, |
4334 | false); |
4335 | return ret_code; |
4336 | } |
4337 | |
4338 | /** |
4339 | * irdma_sc_cfg_iw_fpm() - commits hmc obj cnt values using cqp |
4340 | * command and populates fpm base address in hmc_info |
4341 | * @dev : ptr to irdma_dev struct |
4342 | * @hmc_fn_id: hmc function id |
4343 | */ |
4344 | static int irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev, u8 hmc_fn_id) |
4345 | { |
4346 | struct irdma_hmc_info *hmc_info; |
4347 | struct irdma_hmc_obj_info *obj_info; |
4348 | __le64 *buf; |
4349 | struct irdma_dma_mem commit_fpm_mem; |
4350 | int ret_code = 0; |
4351 | u8 wait_type; |
4352 | |
4353 | hmc_info = dev->hmc_info; |
4354 | obj_info = hmc_info->hmc_obj; |
4355 | buf = dev->fpm_commit_buf; |
4356 | |
4357 | set_64bit_val(wqe_words: buf, byte_index: 0, val: (u64)obj_info[IRDMA_HMC_IW_QP].cnt); |
4358 | set_64bit_val(wqe_words: buf, byte_index: 8, val: (u64)obj_info[IRDMA_HMC_IW_CQ].cnt); |
4359 | set_64bit_val(wqe_words: buf, byte_index: 16, val: (u64)0); /* RSRVD */ |
4360 | set_64bit_val(wqe_words: buf, byte_index: 24, val: (u64)obj_info[IRDMA_HMC_IW_HTE].cnt); |
4361 | set_64bit_val(wqe_words: buf, byte_index: 32, val: (u64)obj_info[IRDMA_HMC_IW_ARP].cnt); |
4362 | set_64bit_val(wqe_words: buf, byte_index: 40, val: (u64)0); /* RSVD */ |
4363 | set_64bit_val(wqe_words: buf, byte_index: 48, val: (u64)obj_info[IRDMA_HMC_IW_MR].cnt); |
4364 | set_64bit_val(wqe_words: buf, byte_index: 56, val: (u64)obj_info[IRDMA_HMC_IW_XF].cnt); |
4365 | set_64bit_val(wqe_words: buf, byte_index: 64, val: (u64)obj_info[IRDMA_HMC_IW_XFFL].cnt); |
4366 | set_64bit_val(wqe_words: buf, byte_index: 72, val: (u64)obj_info[IRDMA_HMC_IW_Q1].cnt); |
4367 | set_64bit_val(wqe_words: buf, byte_index: 80, val: (u64)obj_info[IRDMA_HMC_IW_Q1FL].cnt); |
4368 | set_64bit_val(wqe_words: buf, byte_index: 88, |
4369 | val: (u64)obj_info[IRDMA_HMC_IW_TIMER].cnt); |
4370 | set_64bit_val(wqe_words: buf, byte_index: 96, |
4371 | val: (u64)obj_info[IRDMA_HMC_IW_FSIMC].cnt); |
4372 | set_64bit_val(wqe_words: buf, byte_index: 104, |
4373 | val: (u64)obj_info[IRDMA_HMC_IW_FSIAV].cnt); |
4374 | set_64bit_val(wqe_words: buf, byte_index: 112, |
4375 | val: (u64)obj_info[IRDMA_HMC_IW_PBLE].cnt); |
4376 | set_64bit_val(wqe_words: buf, byte_index: 120, val: (u64)0); /* RSVD */ |
4377 | set_64bit_val(wqe_words: buf, byte_index: 128, val: (u64)obj_info[IRDMA_HMC_IW_RRF].cnt); |
4378 | set_64bit_val(wqe_words: buf, byte_index: 136, |
4379 | val: (u64)obj_info[IRDMA_HMC_IW_RRFFL].cnt); |
4380 | set_64bit_val(wqe_words: buf, byte_index: 144, val: (u64)obj_info[IRDMA_HMC_IW_HDR].cnt); |
4381 | set_64bit_val(wqe_words: buf, byte_index: 152, val: (u64)obj_info[IRDMA_HMC_IW_MD].cnt); |
4382 | set_64bit_val(wqe_words: buf, byte_index: 160, |
4383 | val: (u64)obj_info[IRDMA_HMC_IW_OOISC].cnt); |
4384 | set_64bit_val(wqe_words: buf, byte_index: 168, |
4385 | val: (u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt); |
4386 | |
4387 | commit_fpm_mem.pa = dev->fpm_commit_buf_pa; |
4388 | commit_fpm_mem.va = dev->fpm_commit_buf; |
4389 | |
4390 | wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS; |
4391 | print_hex_dump_debug("HMC: COMMIT FPM BUFFER" , DUMP_PREFIX_OFFSET, 16, |
4392 | 8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE, |
4393 | false); |
4394 | ret_code = irdma_sc_commit_fpm_val(cqp: dev->cqp, scratch: 0, hmc_fn_id: hmc_info->hmc_fn_id, |
4395 | commit_fpm_mem: &commit_fpm_mem, post_sq: true, wait_type); |
4396 | if (!ret_code) |
4397 | irdma_sc_parse_fpm_commit_buf(dev, buf: dev->fpm_commit_buf, |
4398 | info: hmc_info->hmc_obj, |
4399 | sd: &hmc_info->sd_table.sd_cnt); |
4400 | print_hex_dump_debug("HMC: COMMIT FPM BUFFER" , DUMP_PREFIX_OFFSET, 16, |
4401 | 8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE, |
4402 | false); |
4403 | |
4404 | return ret_code; |
4405 | } |
4406 | |
4407 | /** |
4408 | * cqp_sds_wqe_fill - fill cqp wqe doe sd |
4409 | * @cqp: struct for cqp hw |
4410 | * @info: sd info for wqe |
4411 | * @scratch: u64 saved to be used during cqp completion |
4412 | */ |
4413 | static int cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp, |
4414 | struct irdma_update_sds_info *info, u64 scratch) |
4415 | { |
4416 | u64 data; |
4417 | u64 hdr; |
4418 | __le64 *wqe; |
4419 | int mem_entries, wqe_entries; |
4420 | struct irdma_dma_mem *sdbuf = &cqp->sdbuf; |
4421 | u64 offset = 0; |
4422 | u32 wqe_idx; |
4423 | |
4424 | wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, wqe_idx: &wqe_idx); |
4425 | if (!wqe) |
4426 | return -ENOMEM; |
4427 | |
4428 | wqe_entries = (info->cnt > 3) ? 3 : info->cnt; |
4429 | mem_entries = info->cnt - wqe_entries; |
4430 | |
4431 | if (mem_entries) { |
4432 | offset = wqe_idx * IRDMA_UPDATE_SD_BUFF_SIZE; |
4433 | memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4); |
4434 | |
4435 | data = (u64)sdbuf->pa + offset; |
4436 | } else { |
4437 | data = 0; |
4438 | } |
4439 | data |= FIELD_PREP(IRDMA_CQPSQ_UPESD_HMCFNID, info->hmc_fn_id); |
4440 | set_64bit_val(wqe_words: wqe, byte_index: 16, val: data); |
4441 | |
4442 | switch (wqe_entries) { |
4443 | case 3: |
4444 | set_64bit_val(wqe_words: wqe, byte_index: 48, |
4445 | val: (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[2].cmd) | |
4446 | FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1))); |
4447 | |
4448 | set_64bit_val(wqe_words: wqe, byte_index: 56, val: info->entry[2].data); |
4449 | fallthrough; |
4450 | case 2: |
4451 | set_64bit_val(wqe_words: wqe, byte_index: 32, |
4452 | val: (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[1].cmd) | |
4453 | FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1))); |
4454 | |
4455 | set_64bit_val(wqe_words: wqe, byte_index: 40, val: info->entry[1].data); |
4456 | fallthrough; |
4457 | case 1: |
4458 | set_64bit_val(wqe_words: wqe, byte_index: 0, |
4459 | FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[0].cmd)); |
4460 | |
4461 | set_64bit_val(wqe_words: wqe, byte_index: 8, val: info->entry[0].data); |
4462 | break; |
4463 | default: |
4464 | break; |
4465 | } |
4466 | |
4467 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPDATE_PE_SDS) | |
4468 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | |
4469 | FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_COUNT, mem_entries); |
4470 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
4471 | |
4472 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
4473 | |
4474 | if (mem_entries) |
4475 | print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE Buffer" , |
4476 | DUMP_PREFIX_OFFSET, 16, 8, |
4477 | (char *)sdbuf->va + offset, |
4478 | mem_entries << 4, false); |
4479 | |
4480 | print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE" , DUMP_PREFIX_OFFSET, 16, |
4481 | 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
4482 | |
4483 | return 0; |
4484 | } |
4485 | |
4486 | /** |
4487 | * irdma_update_pe_sds - cqp wqe for sd |
4488 | * @dev: ptr to irdma_dev struct |
4489 | * @info: sd info for sd's |
4490 | * @scratch: u64 saved to be used during cqp completion |
4491 | */ |
4492 | static int irdma_update_pe_sds(struct irdma_sc_dev *dev, |
4493 | struct irdma_update_sds_info *info, u64 scratch) |
4494 | { |
4495 | struct irdma_sc_cqp *cqp = dev->cqp; |
4496 | int ret_code; |
4497 | |
4498 | ret_code = cqp_sds_wqe_fill(cqp, info, scratch); |
4499 | if (!ret_code) |
4500 | irdma_sc_cqp_post_sq(cqp); |
4501 | |
4502 | return ret_code; |
4503 | } |
4504 | |
4505 | /** |
4506 | * irdma_update_sds_noccq - update sd before ccq created |
4507 | * @dev: sc device struct |
4508 | * @info: sd info for sd's |
4509 | */ |
4510 | int irdma_update_sds_noccq(struct irdma_sc_dev *dev, |
4511 | struct irdma_update_sds_info *info) |
4512 | { |
4513 | u32 error, val, tail; |
4514 | struct irdma_sc_cqp *cqp = dev->cqp; |
4515 | int ret_code; |
4516 | |
4517 | ret_code = cqp_sds_wqe_fill(cqp, info, scratch: 0); |
4518 | if (ret_code) |
4519 | return ret_code; |
4520 | |
4521 | irdma_get_cqp_reg_info(cqp, val: &val, tail: &tail, error: &error); |
4522 | |
4523 | irdma_sc_cqp_post_sq(cqp); |
4524 | return irdma_cqp_poll_registers(cqp, tail, |
4525 | count: cqp->dev->hw_attrs.max_done_count); |
4526 | } |
4527 | |
4528 | /** |
4529 | * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages |
4530 | * @cqp: struct for cqp hw |
4531 | * @scratch: u64 saved to be used during cqp completion |
4532 | * @hmc_fn_id: hmc function id |
4533 | * @post_sq: flag for cqp db to ring |
4534 | * @poll_registers: flag to poll register for cqp completion |
4535 | */ |
4536 | int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, |
4537 | u8 hmc_fn_id, bool post_sq, |
4538 | bool poll_registers) |
4539 | { |
4540 | u64 hdr; |
4541 | __le64 *wqe; |
4542 | u32 tail, val, error; |
4543 | |
4544 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
4545 | if (!wqe) |
4546 | return -ENOMEM; |
4547 | |
4548 | set_64bit_val(wqe_words: wqe, byte_index: 16, |
4549 | FIELD_PREP(IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID, hmc_fn_id)); |
4550 | |
4551 | hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, |
4552 | IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED) | |
4553 | FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); |
4554 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
4555 | |
4556 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: hdr); |
4557 | |
4558 | print_hex_dump_debug("WQE: SHMC_PAGES_ALLOCATED WQE" , |
4559 | DUMP_PREFIX_OFFSET, 16, 8, wqe, |
4560 | IRDMA_CQP_WQE_SIZE * 8, false); |
4561 | irdma_get_cqp_reg_info(cqp, val: &val, tail: &tail, error: &error); |
4562 | |
4563 | if (post_sq) { |
4564 | irdma_sc_cqp_post_sq(cqp); |
4565 | if (poll_registers) |
4566 | /* check for cqp sq tail update */ |
4567 | return irdma_cqp_poll_registers(cqp, tail, |
4568 | count: cqp->dev->hw_attrs.max_done_count); |
4569 | else |
4570 | return irdma_sc_poll_for_cqp_op_done(cqp, |
4571 | IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED, |
4572 | NULL); |
4573 | } |
4574 | |
4575 | return 0; |
4576 | } |
4577 | |
4578 | /** |
4579 | * irdma_cqp_ring_full - check if cqp ring is full |
4580 | * @cqp: struct for cqp hw |
4581 | */ |
4582 | static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp) |
4583 | { |
4584 | return IRDMA_RING_FULL_ERR(cqp->sq_ring); |
4585 | } |
4586 | |
4587 | /** |
4588 | * irdma_est_sd - returns approximate number of SDs for HMC |
4589 | * @dev: sc device struct |
4590 | * @hmc_info: hmc structure, size and count for HMC objects |
4591 | */ |
4592 | static u32 irdma_est_sd(struct irdma_sc_dev *dev, |
4593 | struct irdma_hmc_info *hmc_info) |
4594 | { |
4595 | int i; |
4596 | u64 size = 0; |
4597 | u64 sd; |
4598 | |
4599 | for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) |
4600 | if (i != IRDMA_HMC_IW_PBLE) |
4601 | size += round_up(hmc_info->hmc_obj[i].cnt * |
4602 | hmc_info->hmc_obj[i].size, 512); |
4603 | size += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt * |
4604 | hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512); |
4605 | if (size & 0x1FFFFF) |
4606 | sd = (size >> 21) + 1; /* add 1 for remainder */ |
4607 | else |
4608 | sd = size >> 21; |
4609 | if (sd > 0xFFFFFFFF) { |
4610 | ibdev_dbg(to_ibdev(dev), "HMC: sd overflow[%lld]\n" , sd); |
4611 | sd = 0xFFFFFFFF - 1; |
4612 | } |
4613 | |
4614 | return (u32)sd; |
4615 | } |
4616 | |
4617 | /** |
4618 | * irdma_sc_query_rdma_features_done - poll cqp for query features done |
4619 | * @cqp: struct for cqp hw |
4620 | */ |
4621 | static int irdma_sc_query_rdma_features_done(struct irdma_sc_cqp *cqp) |
4622 | { |
4623 | return irdma_sc_poll_for_cqp_op_done(cqp, |
4624 | IRDMA_CQP_OP_QUERY_RDMA_FEATURES, |
4625 | NULL); |
4626 | } |
4627 | |
4628 | /** |
4629 | * irdma_sc_query_rdma_features - query RDMA features and FW ver |
4630 | * @cqp: struct for cqp hw |
4631 | * @buf: buffer to hold query info |
4632 | * @scratch: u64 saved to be used during cqp completion |
4633 | */ |
4634 | static int irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp, |
4635 | struct irdma_dma_mem *buf, u64 scratch) |
4636 | { |
4637 | __le64 *wqe; |
4638 | u64 temp; |
4639 | |
4640 | wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); |
4641 | if (!wqe) |
4642 | return -ENOMEM; |
4643 | |
4644 | temp = buf->pa; |
4645 | set_64bit_val(wqe_words: wqe, byte_index: 32, val: temp); |
4646 | |
4647 | temp = FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID, |
4648 | cqp->polarity) | |
4649 | FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf->size) | |
4650 | FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_QUERY_RDMA_FEATURES); |
4651 | dma_wmb(); /* make sure WQE is written before valid bit is set */ |
4652 | |
4653 | set_64bit_val(wqe_words: wqe, byte_index: 24, val: temp); |
4654 | |
4655 | print_hex_dump_debug("WQE: QUERY RDMA FEATURES" , DUMP_PREFIX_OFFSET, |
4656 | 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); |
4657 | irdma_sc_cqp_post_sq(cqp); |
4658 | |
4659 | return 0; |
4660 | } |
4661 | |
4662 | /** |
4663 | * irdma_get_rdma_features - get RDMA features |
4664 | * @dev: sc device struct |
4665 | */ |
4666 | int irdma_get_rdma_features(struct irdma_sc_dev *dev) |
4667 | { |
4668 | int ret_code; |
4669 | struct irdma_dma_mem feat_buf; |
4670 | u64 temp; |
4671 | u16 byte_idx, feat_type, feat_cnt, feat_idx; |
4672 | |
4673 | feat_buf.size = ALIGN(IRDMA_FEATURE_BUF_SIZE, |
4674 | IRDMA_FEATURE_BUF_ALIGNMENT); |
4675 | feat_buf.va = dma_alloc_coherent(dev: dev->hw->device, size: feat_buf.size, |
4676 | dma_handle: &feat_buf.pa, GFP_KERNEL); |
4677 | if (!feat_buf.va) |
4678 | return -ENOMEM; |
4679 | |
4680 | ret_code = irdma_sc_query_rdma_features(cqp: dev->cqp, buf: &feat_buf, scratch: 0); |
4681 | if (!ret_code) |
4682 | ret_code = irdma_sc_query_rdma_features_done(cqp: dev->cqp); |
4683 | if (ret_code) |
4684 | goto exit; |
4685 | |
4686 | get_64bit_val(wqe_words: feat_buf.va, byte_index: 0, val: &temp); |
4687 | feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp); |
4688 | if (feat_cnt < 2) { |
4689 | ret_code = -EINVAL; |
4690 | goto exit; |
4691 | } else if (feat_cnt > IRDMA_MAX_FEATURES) { |
4692 | ibdev_dbg(to_ibdev(dev), |
4693 | "DEV: feature buf size insufficient, retrying with larger buffer\n" ); |
4694 | dma_free_coherent(dev: dev->hw->device, size: feat_buf.size, cpu_addr: feat_buf.va, |
4695 | dma_handle: feat_buf.pa); |
4696 | feat_buf.va = NULL; |
4697 | feat_buf.size = ALIGN(8 * feat_cnt, |
4698 | IRDMA_FEATURE_BUF_ALIGNMENT); |
4699 | feat_buf.va = dma_alloc_coherent(dev: dev->hw->device, |
4700 | size: feat_buf.size, dma_handle: &feat_buf.pa, |
4701 | GFP_KERNEL); |
4702 | if (!feat_buf.va) |
4703 | return -ENOMEM; |
4704 | |
4705 | ret_code = irdma_sc_query_rdma_features(cqp: dev->cqp, buf: &feat_buf, scratch: 0); |
4706 | if (!ret_code) |
4707 | ret_code = irdma_sc_query_rdma_features_done(cqp: dev->cqp); |
4708 | if (ret_code) |
4709 | goto exit; |
4710 | |
4711 | get_64bit_val(wqe_words: feat_buf.va, byte_index: 0, val: &temp); |
4712 | feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp); |
4713 | if (feat_cnt < 2) { |
4714 | ret_code = -EINVAL; |
4715 | goto exit; |
4716 | } |
4717 | } |
4718 | |
4719 | print_hex_dump_debug("WQE: QUERY RDMA FEATURES" , DUMP_PREFIX_OFFSET, |
4720 | 16, 8, feat_buf.va, feat_cnt * 8, false); |
4721 | |
4722 | for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES); |
4723 | feat_idx++, byte_idx += 8) { |
4724 | get_64bit_val(wqe_words: feat_buf.va, byte_index: byte_idx, val: &temp); |
4725 | feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp); |
4726 | if (feat_type >= IRDMA_MAX_FEATURES) { |
4727 | ibdev_dbg(to_ibdev(dev), |
4728 | "DEV: found unrecognized feature type %d\n" , |
4729 | feat_type); |
4730 | continue; |
4731 | } |
4732 | dev->feature_info[feat_type] = temp; |
4733 | } |
4734 | exit: |
4735 | dma_free_coherent(dev: dev->hw->device, size: feat_buf.size, cpu_addr: feat_buf.va, |
4736 | dma_handle: feat_buf.pa); |
4737 | feat_buf.va = NULL; |
4738 | return ret_code; |
4739 | } |
4740 | |
4741 | static u32 irdma_q1_cnt(struct irdma_sc_dev *dev, |
4742 | struct irdma_hmc_info *hmc_info, u32 qpwanted) |
4743 | { |
4744 | u32 q1_cnt; |
4745 | |
4746 | if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { |
4747 | q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted); |
4748 | } else { |
4749 | if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) |
4750 | q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted + 512); |
4751 | else |
4752 | q1_cnt = dev->hw_attrs.max_hw_ird * 2 * qpwanted; |
4753 | } |
4754 | |
4755 | return q1_cnt; |
4756 | } |
4757 | |
4758 | static void cfg_fpm_value_gen_1(struct irdma_sc_dev *dev, |
4759 | struct irdma_hmc_info *hmc_info, u32 qpwanted) |
4760 | { |
4761 | hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = roundup_pow_of_two(qpwanted * dev->hw_attrs.max_hw_wqes); |
4762 | } |
4763 | |
4764 | static void cfg_fpm_value_gen_2(struct irdma_sc_dev *dev, |
4765 | struct irdma_hmc_info *hmc_info, u32 qpwanted) |
4766 | { |
4767 | struct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc; |
4768 | |
4769 | hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = |
4770 | 4 * hmc_fpm_misc->xf_block_size * qpwanted; |
4771 | |
4772 | hmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted; |
4773 | |
4774 | if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt) |
4775 | hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted; |
4776 | if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt) |
4777 | hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt = |
4778 | hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt / |
4779 | hmc_fpm_misc->rrf_block_size; |
4780 | if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt) |
4781 | hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted; |
4782 | if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt) |
4783 | hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt = |
4784 | hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt / |
4785 | hmc_fpm_misc->ooiscf_block_size; |
4786 | } |
4787 | |
4788 | /** |
4789 | * irdma_cfg_fpm_val - configure HMC objects |
4790 | * @dev: sc device struct |
4791 | * @qp_count: desired qp count |
4792 | */ |
4793 | int irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count) |
4794 | { |
4795 | struct irdma_virt_mem virt_mem; |
4796 | u32 i, mem_size; |
4797 | u32 qpwanted, mrwanted, pblewanted; |
4798 | u32 powerof2, hte; |
4799 | u32 sd_needed; |
4800 | u32 sd_diff; |
4801 | u32 loop_count = 0; |
4802 | struct irdma_hmc_info *hmc_info; |
4803 | struct irdma_hmc_fpm_misc *hmc_fpm_misc; |
4804 | int ret_code = 0; |
4805 | |
4806 | hmc_info = dev->hmc_info; |
4807 | hmc_fpm_misc = &dev->hmc_fpm_misc; |
4808 | |
4809 | ret_code = irdma_sc_init_iw_hmc(dev, hmc_fn_id: dev->hmc_fn_id); |
4810 | if (ret_code) { |
4811 | ibdev_dbg(to_ibdev(dev), |
4812 | "HMC: irdma_sc_init_iw_hmc returned error_code = %d\n" , |
4813 | ret_code); |
4814 | return ret_code; |
4815 | } |
4816 | |
4817 | for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) |
4818 | hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt; |
4819 | sd_needed = irdma_est_sd(dev, hmc_info); |
4820 | ibdev_dbg(to_ibdev(dev), |
4821 | "HMC: FW max resources sd_needed[%08d] first_sd_index[%04d]\n" , |
4822 | sd_needed, hmc_info->first_sd_index); |
4823 | ibdev_dbg(to_ibdev(dev), "HMC: sd count %d where max sd is %d\n" , |
4824 | hmc_info->sd_table.sd_cnt, hmc_fpm_misc->max_sds); |
4825 | |
4826 | qpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt); |
4827 | |
4828 | powerof2 = 1; |
4829 | while (powerof2 <= qpwanted) |
4830 | powerof2 *= 2; |
4831 | powerof2 /= 2; |
4832 | qpwanted = powerof2; |
4833 | |
4834 | mrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt; |
4835 | pblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt; |
4836 | |
4837 | ibdev_dbg(to_ibdev(dev), |
4838 | "HMC: req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d, mc=%d, av=%d\n" , |
4839 | qp_count, hmc_fpm_misc->max_sds, |
4840 | hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt, |
4841 | hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt, |
4842 | hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt, |
4843 | hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt, |
4844 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt, |
4845 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt); |
4846 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt = |
4847 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt; |
4848 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt = |
4849 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt; |
4850 | hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt = |
4851 | hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt; |
4852 | |
4853 | hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1; |
4854 | |
4855 | while (irdma_q1_cnt(dev, hmc_info, qpwanted) > hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].max_cnt) |
4856 | qpwanted /= 2; |
4857 | |
4858 | do { |
4859 | ++loop_count; |
4860 | hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted; |
4861 | hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt = |
4862 | min(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt); |
4863 | hmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */ |
4864 | hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted; |
4865 | |
4866 | hte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512); |
4867 | powerof2 = 1; |
4868 | while (powerof2 < hte) |
4869 | powerof2 *= 2; |
4870 | hmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt = |
4871 | powerof2 * hmc_fpm_misc->ht_multiplier; |
4872 | if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) |
4873 | cfg_fpm_value_gen_1(dev, hmc_info, qpwanted); |
4874 | else |
4875 | cfg_fpm_value_gen_2(dev, hmc_info, qpwanted); |
4876 | |
4877 | hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = irdma_q1_cnt(dev, hmc_info, qpwanted); |
4878 | hmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt = |
4879 | hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size; |
4880 | hmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt = |
4881 | hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size; |
4882 | hmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt = |
4883 | (round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket; |
4884 | |
4885 | hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted; |
4886 | sd_needed = irdma_est_sd(dev, hmc_info); |
4887 | ibdev_dbg(to_ibdev(dev), |
4888 | "HMC: sd_needed = %d, hmc_fpm_misc->max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n" , |
4889 | sd_needed, hmc_fpm_misc->max_sds, mrwanted, |
4890 | pblewanted, qpwanted); |
4891 | |
4892 | /* Do not reduce resources further. All objects fit with max SDs */ |
4893 | if (sd_needed <= hmc_fpm_misc->max_sds) |
4894 | break; |
4895 | |
4896 | sd_diff = sd_needed - hmc_fpm_misc->max_sds; |
4897 | if (sd_diff > 128) { |
4898 | if (!(loop_count % 2) && qpwanted > 128) { |
4899 | qpwanted /= 2; |
4900 | } else { |
4901 | mrwanted /= 2; |
4902 | pblewanted /= 2; |
4903 | } |
4904 | continue; |
4905 | } |
4906 | if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF && |
4907 | pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) { |
4908 | pblewanted -= 256 * FPM_MULTIPLIER * sd_diff; |
4909 | continue; |
4910 | } else if (pblewanted > (100 * FPM_MULTIPLIER)) { |
4911 | pblewanted -= 10 * FPM_MULTIPLIER; |
4912 | } else if (pblewanted > FPM_MULTIPLIER) { |
4913 | pblewanted -= FPM_MULTIPLIER; |
4914 | } else if (qpwanted <= 128) { |
4915 | if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256) |
4916 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2; |
4917 | if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256) |
4918 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2; |
4919 | } |
4920 | if (mrwanted > FPM_MULTIPLIER) |
4921 | mrwanted -= FPM_MULTIPLIER; |
4922 | if (!(loop_count % 10) && qpwanted > 128) { |
4923 | qpwanted /= 2; |
4924 | if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256) |
4925 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2; |
4926 | } |
4927 | } while (loop_count < 2000); |
4928 | |
4929 | if (sd_needed > hmc_fpm_misc->max_sds) { |
4930 | ibdev_dbg(to_ibdev(dev), |
4931 | "HMC: cfg_fpm failed loop_cnt=%d, sd_needed=%d, max sd count %d\n" , |
4932 | loop_count, sd_needed, hmc_info->sd_table.sd_cnt); |
4933 | return -EINVAL; |
4934 | } |
4935 | |
4936 | if (loop_count > 1 && sd_needed < hmc_fpm_misc->max_sds) { |
4937 | pblewanted += (hmc_fpm_misc->max_sds - sd_needed) * 256 * |
4938 | FPM_MULTIPLIER; |
4939 | hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted; |
4940 | sd_needed = irdma_est_sd(dev, hmc_info); |
4941 | } |
4942 | |
4943 | ibdev_dbg(to_ibdev(dev), |
4944 | "HMC: loop_cnt=%d, sd_needed=%d, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d, mc=%d, ah=%d, max sd count %d, first sd index %d\n" , |
4945 | loop_count, sd_needed, |
4946 | hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt, |
4947 | hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt, |
4948 | hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt, |
4949 | hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt, |
4950 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, |
4951 | hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt, |
4952 | hmc_info->sd_table.sd_cnt, hmc_info->first_sd_index); |
4953 | |
4954 | ret_code = irdma_sc_cfg_iw_fpm(dev, hmc_fn_id: dev->hmc_fn_id); |
4955 | if (ret_code) { |
4956 | ibdev_dbg(to_ibdev(dev), |
4957 | "HMC: cfg_iw_fpm returned error_code[x%08X]\n" , |
4958 | readl(dev->hw_regs[IRDMA_CQPERRCODES])); |
4959 | return ret_code; |
4960 | } |
4961 | |
4962 | mem_size = sizeof(struct irdma_hmc_sd_entry) * |
4963 | (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1); |
4964 | virt_mem.size = mem_size; |
4965 | virt_mem.va = kzalloc(size: virt_mem.size, GFP_KERNEL); |
4966 | if (!virt_mem.va) { |
4967 | ibdev_dbg(to_ibdev(dev), |
4968 | "HMC: failed to allocate memory for sd_entry buffer\n" ); |
4969 | return -ENOMEM; |
4970 | } |
4971 | hmc_info->sd_table.sd_entry = virt_mem.va; |
4972 | |
4973 | return ret_code; |
4974 | } |
4975 | |
4976 | /** |
4977 | * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available |
4978 | * @dev: rdma device |
4979 | * @pcmdinfo: cqp command info |
4980 | */ |
4981 | static int irdma_exec_cqp_cmd(struct irdma_sc_dev *dev, |
4982 | struct cqp_cmds_info *pcmdinfo) |
4983 | { |
4984 | int status; |
4985 | struct irdma_dma_mem val_mem; |
4986 | bool alloc = false; |
4987 | |
4988 | dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++; |
4989 | switch (pcmdinfo->cqp_cmd) { |
4990 | case IRDMA_OP_CEQ_DESTROY: |
4991 | status = irdma_sc_ceq_destroy(ceq: pcmdinfo->in.u.ceq_destroy.ceq, |
4992 | scratch: pcmdinfo->in.u.ceq_destroy.scratch, |
4993 | post_sq: pcmdinfo->post_sq); |
4994 | break; |
4995 | case IRDMA_OP_AEQ_DESTROY: |
4996 | status = irdma_sc_aeq_destroy(aeq: pcmdinfo->in.u.aeq_destroy.aeq, |
4997 | scratch: pcmdinfo->in.u.aeq_destroy.scratch, |
4998 | post_sq: pcmdinfo->post_sq); |
4999 | |
5000 | break; |
5001 | case IRDMA_OP_CEQ_CREATE: |
5002 | status = irdma_sc_ceq_create(ceq: pcmdinfo->in.u.ceq_create.ceq, |
5003 | scratch: pcmdinfo->in.u.ceq_create.scratch, |
5004 | post_sq: pcmdinfo->post_sq); |
5005 | break; |
5006 | case IRDMA_OP_AEQ_CREATE: |
5007 | status = irdma_sc_aeq_create(aeq: pcmdinfo->in.u.aeq_create.aeq, |
5008 | scratch: pcmdinfo->in.u.aeq_create.scratch, |
5009 | post_sq: pcmdinfo->post_sq); |
5010 | break; |
5011 | case IRDMA_OP_QP_UPLOAD_CONTEXT: |
5012 | status = irdma_sc_qp_upload_context(dev: pcmdinfo->in.u.qp_upload_context.dev, |
5013 | info: &pcmdinfo->in.u.qp_upload_context.info, |
5014 | scratch: pcmdinfo->in.u.qp_upload_context.scratch, |
5015 | post_sq: pcmdinfo->post_sq); |
5016 | break; |
5017 | case IRDMA_OP_CQ_CREATE: |
5018 | status = irdma_sc_cq_create(cq: pcmdinfo->in.u.cq_create.cq, |
5019 | scratch: pcmdinfo->in.u.cq_create.scratch, |
5020 | check_overflow: pcmdinfo->in.u.cq_create.check_overflow, |
5021 | post_sq: pcmdinfo->post_sq); |
5022 | break; |
5023 | case IRDMA_OP_CQ_MODIFY: |
5024 | status = irdma_sc_cq_modify(cq: pcmdinfo->in.u.cq_modify.cq, |
5025 | info: &pcmdinfo->in.u.cq_modify.info, |
5026 | scratch: pcmdinfo->in.u.cq_modify.scratch, |
5027 | post_sq: pcmdinfo->post_sq); |
5028 | break; |
5029 | case IRDMA_OP_CQ_DESTROY: |
5030 | status = irdma_sc_cq_destroy(cq: pcmdinfo->in.u.cq_destroy.cq, |
5031 | scratch: pcmdinfo->in.u.cq_destroy.scratch, |
5032 | post_sq: pcmdinfo->post_sq); |
5033 | break; |
5034 | case IRDMA_OP_QP_FLUSH_WQES: |
5035 | status = irdma_sc_qp_flush_wqes(qp: pcmdinfo->in.u.qp_flush_wqes.qp, |
5036 | info: &pcmdinfo->in.u.qp_flush_wqes.info, |
5037 | scratch: pcmdinfo->in.u.qp_flush_wqes.scratch, |
5038 | post_sq: pcmdinfo->post_sq); |
5039 | break; |
5040 | case IRDMA_OP_GEN_AE: |
5041 | status = irdma_sc_gen_ae(qp: pcmdinfo->in.u.gen_ae.qp, |
5042 | info: &pcmdinfo->in.u.gen_ae.info, |
5043 | scratch: pcmdinfo->in.u.gen_ae.scratch, |
5044 | post_sq: pcmdinfo->post_sq); |
5045 | break; |
5046 | case IRDMA_OP_MANAGE_PUSH_PAGE: |
5047 | status = irdma_sc_manage_push_page(cqp: pcmdinfo->in.u.manage_push_page.cqp, |
5048 | info: &pcmdinfo->in.u.manage_push_page.info, |
5049 | scratch: pcmdinfo->in.u.manage_push_page.scratch, |
5050 | post_sq: pcmdinfo->post_sq); |
5051 | break; |
5052 | case IRDMA_OP_UPDATE_PE_SDS: |
5053 | status = irdma_update_pe_sds(dev: pcmdinfo->in.u.update_pe_sds.dev, |
5054 | info: &pcmdinfo->in.u.update_pe_sds.info, |
5055 | scratch: pcmdinfo->in.u.update_pe_sds.scratch); |
5056 | break; |
5057 | case IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE: |
5058 | /* switch to calling through the call table */ |
5059 | status = |
5060 | irdma_sc_manage_hmc_pm_func_table(cqp: pcmdinfo->in.u.manage_hmc_pm.dev->cqp, |
5061 | info: &pcmdinfo->in.u.manage_hmc_pm.info, |
5062 | scratch: pcmdinfo->in.u.manage_hmc_pm.scratch, |
5063 | post_sq: true); |
5064 | break; |
5065 | case IRDMA_OP_SUSPEND: |
5066 | status = irdma_sc_suspend_qp(cqp: pcmdinfo->in.u.suspend_resume.cqp, |
5067 | qp: pcmdinfo->in.u.suspend_resume.qp, |
5068 | scratch: pcmdinfo->in.u.suspend_resume.scratch); |
5069 | break; |
5070 | case IRDMA_OP_RESUME: |
5071 | status = irdma_sc_resume_qp(cqp: pcmdinfo->in.u.suspend_resume.cqp, |
5072 | qp: pcmdinfo->in.u.suspend_resume.qp, |
5073 | scratch: pcmdinfo->in.u.suspend_resume.scratch); |
5074 | break; |
5075 | case IRDMA_OP_QUERY_FPM_VAL: |
5076 | val_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa; |
5077 | val_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va; |
5078 | status = irdma_sc_query_fpm_val(cqp: pcmdinfo->in.u.query_fpm_val.cqp, |
5079 | scratch: pcmdinfo->in.u.query_fpm_val.scratch, |
5080 | hmc_fn_id: pcmdinfo->in.u.query_fpm_val.hmc_fn_id, |
5081 | query_fpm_mem: &val_mem, post_sq: true, IRDMA_CQP_WAIT_EVENT); |
5082 | break; |
5083 | case IRDMA_OP_COMMIT_FPM_VAL: |
5084 | val_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa; |
5085 | val_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va; |
5086 | status = irdma_sc_commit_fpm_val(cqp: pcmdinfo->in.u.commit_fpm_val.cqp, |
5087 | scratch: pcmdinfo->in.u.commit_fpm_val.scratch, |
5088 | hmc_fn_id: pcmdinfo->in.u.commit_fpm_val.hmc_fn_id, |
5089 | commit_fpm_mem: &val_mem, |
5090 | post_sq: true, |
5091 | IRDMA_CQP_WAIT_EVENT); |
5092 | break; |
5093 | case IRDMA_OP_STATS_ALLOCATE: |
5094 | alloc = true; |
5095 | fallthrough; |
5096 | case IRDMA_OP_STATS_FREE: |
5097 | status = irdma_sc_manage_stats_inst(cqp: pcmdinfo->in.u.stats_manage.cqp, |
5098 | info: &pcmdinfo->in.u.stats_manage.info, |
5099 | alloc, |
5100 | scratch: pcmdinfo->in.u.stats_manage.scratch); |
5101 | break; |
5102 | case IRDMA_OP_STATS_GATHER: |
5103 | status = irdma_sc_gather_stats(cqp: pcmdinfo->in.u.stats_gather.cqp, |
5104 | info: &pcmdinfo->in.u.stats_gather.info, |
5105 | scratch: pcmdinfo->in.u.stats_gather.scratch); |
5106 | break; |
5107 | case IRDMA_OP_WS_MODIFY_NODE: |
5108 | status = irdma_sc_manage_ws_node(cqp: pcmdinfo->in.u.ws_node.cqp, |
5109 | info: &pcmdinfo->in.u.ws_node.info, |
5110 | node_op: IRDMA_MODIFY_NODE, |
5111 | scratch: pcmdinfo->in.u.ws_node.scratch); |
5112 | break; |
5113 | case IRDMA_OP_WS_DELETE_NODE: |
5114 | status = irdma_sc_manage_ws_node(cqp: pcmdinfo->in.u.ws_node.cqp, |
5115 | info: &pcmdinfo->in.u.ws_node.info, |
5116 | node_op: IRDMA_DEL_NODE, |
5117 | scratch: pcmdinfo->in.u.ws_node.scratch); |
5118 | break; |
5119 | case IRDMA_OP_WS_ADD_NODE: |
5120 | status = irdma_sc_manage_ws_node(cqp: pcmdinfo->in.u.ws_node.cqp, |
5121 | info: &pcmdinfo->in.u.ws_node.info, |
5122 | node_op: IRDMA_ADD_NODE, |
5123 | scratch: pcmdinfo->in.u.ws_node.scratch); |
5124 | break; |
5125 | case IRDMA_OP_SET_UP_MAP: |
5126 | status = irdma_sc_set_up_map(cqp: pcmdinfo->in.u.up_map.cqp, |
5127 | info: &pcmdinfo->in.u.up_map.info, |
5128 | scratch: pcmdinfo->in.u.up_map.scratch); |
5129 | break; |
5130 | case IRDMA_OP_QUERY_RDMA_FEATURES: |
5131 | status = irdma_sc_query_rdma_features(cqp: pcmdinfo->in.u.query_rdma.cqp, |
5132 | buf: &pcmdinfo->in.u.query_rdma.query_buff_mem, |
5133 | scratch: pcmdinfo->in.u.query_rdma.scratch); |
5134 | break; |
5135 | case IRDMA_OP_DELETE_ARP_CACHE_ENTRY: |
5136 | status = irdma_sc_del_arp_cache_entry(cqp: pcmdinfo->in.u.del_arp_cache_entry.cqp, |
5137 | scratch: pcmdinfo->in.u.del_arp_cache_entry.scratch, |
5138 | arp_index: pcmdinfo->in.u.del_arp_cache_entry.arp_index, |
5139 | post_sq: pcmdinfo->post_sq); |
5140 | break; |
5141 | case IRDMA_OP_MANAGE_APBVT_ENTRY: |
5142 | status = irdma_sc_manage_apbvt_entry(cqp: pcmdinfo->in.u.manage_apbvt_entry.cqp, |
5143 | info: &pcmdinfo->in.u.manage_apbvt_entry.info, |
5144 | scratch: pcmdinfo->in.u.manage_apbvt_entry.scratch, |
5145 | post_sq: pcmdinfo->post_sq); |
5146 | break; |
5147 | case IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY: |
5148 | status = irdma_sc_manage_qhash_table_entry(cqp: pcmdinfo->in.u.manage_qhash_table_entry.cqp, |
5149 | info: &pcmdinfo->in.u.manage_qhash_table_entry.info, |
5150 | scratch: pcmdinfo->in.u.manage_qhash_table_entry.scratch, |
5151 | post_sq: pcmdinfo->post_sq); |
5152 | break; |
5153 | case IRDMA_OP_QP_MODIFY: |
5154 | status = irdma_sc_qp_modify(qp: pcmdinfo->in.u.qp_modify.qp, |
5155 | info: &pcmdinfo->in.u.qp_modify.info, |
5156 | scratch: pcmdinfo->in.u.qp_modify.scratch, |
5157 | post_sq: pcmdinfo->post_sq); |
5158 | break; |
5159 | case IRDMA_OP_QP_CREATE: |
5160 | status = irdma_sc_qp_create(qp: pcmdinfo->in.u.qp_create.qp, |
5161 | info: &pcmdinfo->in.u.qp_create.info, |
5162 | scratch: pcmdinfo->in.u.qp_create.scratch, |
5163 | post_sq: pcmdinfo->post_sq); |
5164 | break; |
5165 | case IRDMA_OP_QP_DESTROY: |
5166 | status = irdma_sc_qp_destroy(qp: pcmdinfo->in.u.qp_destroy.qp, |
5167 | scratch: pcmdinfo->in.u.qp_destroy.scratch, |
5168 | remove_hash_idx: pcmdinfo->in.u.qp_destroy.remove_hash_idx, |
5169 | ignore_mw_bnd: pcmdinfo->in.u.qp_destroy.ignore_mw_bnd, |
5170 | post_sq: pcmdinfo->post_sq); |
5171 | break; |
5172 | case IRDMA_OP_ALLOC_STAG: |
5173 | status = irdma_sc_alloc_stag(dev: pcmdinfo->in.u.alloc_stag.dev, |
5174 | info: &pcmdinfo->in.u.alloc_stag.info, |
5175 | scratch: pcmdinfo->in.u.alloc_stag.scratch, |
5176 | post_sq: pcmdinfo->post_sq); |
5177 | break; |
5178 | case IRDMA_OP_MR_REG_NON_SHARED: |
5179 | status = irdma_sc_mr_reg_non_shared(dev: pcmdinfo->in.u.mr_reg_non_shared.dev, |
5180 | info: &pcmdinfo->in.u.mr_reg_non_shared.info, |
5181 | scratch: pcmdinfo->in.u.mr_reg_non_shared.scratch, |
5182 | post_sq: pcmdinfo->post_sq); |
5183 | break; |
5184 | case IRDMA_OP_DEALLOC_STAG: |
5185 | status = irdma_sc_dealloc_stag(dev: pcmdinfo->in.u.dealloc_stag.dev, |
5186 | info: &pcmdinfo->in.u.dealloc_stag.info, |
5187 | scratch: pcmdinfo->in.u.dealloc_stag.scratch, |
5188 | post_sq: pcmdinfo->post_sq); |
5189 | break; |
5190 | case IRDMA_OP_MW_ALLOC: |
5191 | status = irdma_sc_mw_alloc(dev: pcmdinfo->in.u.mw_alloc.dev, |
5192 | info: &pcmdinfo->in.u.mw_alloc.info, |
5193 | scratch: pcmdinfo->in.u.mw_alloc.scratch, |
5194 | post_sq: pcmdinfo->post_sq); |
5195 | break; |
5196 | case IRDMA_OP_ADD_ARP_CACHE_ENTRY: |
5197 | status = irdma_sc_add_arp_cache_entry(cqp: pcmdinfo->in.u.add_arp_cache_entry.cqp, |
5198 | info: &pcmdinfo->in.u.add_arp_cache_entry.info, |
5199 | scratch: pcmdinfo->in.u.add_arp_cache_entry.scratch, |
5200 | post_sq: pcmdinfo->post_sq); |
5201 | break; |
5202 | case IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY: |
5203 | status = irdma_sc_alloc_local_mac_entry(cqp: pcmdinfo->in.u.alloc_local_mac_entry.cqp, |
5204 | scratch: pcmdinfo->in.u.alloc_local_mac_entry.scratch, |
5205 | post_sq: pcmdinfo->post_sq); |
5206 | break; |
5207 | case IRDMA_OP_ADD_LOCAL_MAC_ENTRY: |
5208 | status = irdma_sc_add_local_mac_entry(cqp: pcmdinfo->in.u.add_local_mac_entry.cqp, |
5209 | info: &pcmdinfo->in.u.add_local_mac_entry.info, |
5210 | scratch: pcmdinfo->in.u.add_local_mac_entry.scratch, |
5211 | post_sq: pcmdinfo->post_sq); |
5212 | break; |
5213 | case IRDMA_OP_DELETE_LOCAL_MAC_ENTRY: |
5214 | status = irdma_sc_del_local_mac_entry(cqp: pcmdinfo->in.u.del_local_mac_entry.cqp, |
5215 | scratch: pcmdinfo->in.u.del_local_mac_entry.scratch, |
5216 | entry_idx: pcmdinfo->in.u.del_local_mac_entry.entry_idx, |
5217 | ignore_ref_count: pcmdinfo->in.u.del_local_mac_entry.ignore_ref_count, |
5218 | post_sq: pcmdinfo->post_sq); |
5219 | break; |
5220 | case IRDMA_OP_AH_CREATE: |
5221 | status = irdma_sc_create_ah(cqp: pcmdinfo->in.u.ah_create.cqp, |
5222 | info: &pcmdinfo->in.u.ah_create.info, |
5223 | scratch: pcmdinfo->in.u.ah_create.scratch); |
5224 | break; |
5225 | case IRDMA_OP_AH_DESTROY: |
5226 | status = irdma_sc_destroy_ah(cqp: pcmdinfo->in.u.ah_destroy.cqp, |
5227 | info: &pcmdinfo->in.u.ah_destroy.info, |
5228 | scratch: pcmdinfo->in.u.ah_destroy.scratch); |
5229 | break; |
5230 | case IRDMA_OP_MC_CREATE: |
5231 | status = irdma_sc_create_mcast_grp(cqp: pcmdinfo->in.u.mc_create.cqp, |
5232 | info: &pcmdinfo->in.u.mc_create.info, |
5233 | scratch: pcmdinfo->in.u.mc_create.scratch); |
5234 | break; |
5235 | case IRDMA_OP_MC_DESTROY: |
5236 | status = irdma_sc_destroy_mcast_grp(cqp: pcmdinfo->in.u.mc_destroy.cqp, |
5237 | info: &pcmdinfo->in.u.mc_destroy.info, |
5238 | scratch: pcmdinfo->in.u.mc_destroy.scratch); |
5239 | break; |
5240 | case IRDMA_OP_MC_MODIFY: |
5241 | status = irdma_sc_modify_mcast_grp(cqp: pcmdinfo->in.u.mc_modify.cqp, |
5242 | info: &pcmdinfo->in.u.mc_modify.info, |
5243 | scratch: pcmdinfo->in.u.mc_modify.scratch); |
5244 | break; |
5245 | default: |
5246 | status = -EOPNOTSUPP; |
5247 | break; |
5248 | } |
5249 | |
5250 | return status; |
5251 | } |
5252 | |
5253 | /** |
5254 | * irdma_process_cqp_cmd - process all cqp commands |
5255 | * @dev: sc device struct |
5256 | * @pcmdinfo: cqp command info |
5257 | */ |
5258 | int irdma_process_cqp_cmd(struct irdma_sc_dev *dev, |
5259 | struct cqp_cmds_info *pcmdinfo) |
5260 | { |
5261 | int status = 0; |
5262 | unsigned long flags; |
5263 | |
5264 | spin_lock_irqsave(&dev->cqp_lock, flags); |
5265 | if (list_empty(head: &dev->cqp_cmd_head) && !irdma_cqp_ring_full(cqp: dev->cqp)) |
5266 | status = irdma_exec_cqp_cmd(dev, pcmdinfo); |
5267 | else |
5268 | list_add_tail(new: &pcmdinfo->cqp_cmd_entry, head: &dev->cqp_cmd_head); |
5269 | spin_unlock_irqrestore(lock: &dev->cqp_lock, flags); |
5270 | return status; |
5271 | } |
5272 | |
5273 | /** |
5274 | * irdma_process_bh - called from tasklet for cqp list |
5275 | * @dev: sc device struct |
5276 | */ |
5277 | int irdma_process_bh(struct irdma_sc_dev *dev) |
5278 | { |
5279 | int status = 0; |
5280 | struct cqp_cmds_info *pcmdinfo; |
5281 | unsigned long flags; |
5282 | |
5283 | spin_lock_irqsave(&dev->cqp_lock, flags); |
5284 | while (!list_empty(head: &dev->cqp_cmd_head) && |
5285 | !irdma_cqp_ring_full(cqp: dev->cqp)) { |
5286 | pcmdinfo = (struct cqp_cmds_info *)irdma_remove_cqp_head(dev); |
5287 | status = irdma_exec_cqp_cmd(dev, pcmdinfo); |
5288 | if (status) |
5289 | break; |
5290 | } |
5291 | spin_unlock_irqrestore(lock: &dev->cqp_lock, flags); |
5292 | return status; |
5293 | } |
5294 | |
5295 | /** |
5296 | * irdma_cfg_aeq- Configure AEQ interrupt |
5297 | * @dev: pointer to the device structure |
5298 | * @idx: vector index |
5299 | * @enable: True to enable, False disables |
5300 | */ |
5301 | void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable) |
5302 | { |
5303 | u32 reg_val; |
5304 | |
5305 | reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) | |
5306 | FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) | |
5307 | FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, 3); |
5308 | writel(val: reg_val, addr: dev->hw_regs[IRDMA_PFINT_AEQCTL]); |
5309 | } |
5310 | |
5311 | /** |
5312 | * sc_vsi_update_stats - Update statistics |
5313 | * @vsi: sc_vsi instance to update |
5314 | */ |
5315 | void sc_vsi_update_stats(struct irdma_sc_vsi *vsi) |
5316 | { |
5317 | struct irdma_gather_stats *gather_stats; |
5318 | struct irdma_gather_stats *last_gather_stats; |
5319 | |
5320 | gather_stats = vsi->pestat->gather_info.gather_stats_va; |
5321 | last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va; |
5322 | irdma_update_stats(hw_stats: &vsi->pestat->hw_stats, gather_stats, |
5323 | last_gather_stats, map: vsi->dev->hw_stats_map, |
5324 | max_stat_idx: vsi->dev->hw_attrs.max_stat_idx); |
5325 | } |
5326 | |
5327 | /** |
5328 | * irdma_wait_pe_ready - Check if firmware is ready |
5329 | * @dev: provides access to registers |
5330 | */ |
5331 | static int irdma_wait_pe_ready(struct irdma_sc_dev *dev) |
5332 | { |
5333 | u32 statuscpu0; |
5334 | u32 statuscpu1; |
5335 | u32 statuscpu2; |
5336 | u32 retrycount = 0; |
5337 | |
5338 | do { |
5339 | statuscpu0 = readl(addr: dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]); |
5340 | statuscpu1 = readl(addr: dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]); |
5341 | statuscpu2 = readl(addr: dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]); |
5342 | if (statuscpu0 == 0x80 && statuscpu1 == 0x80 && |
5343 | statuscpu2 == 0x80) |
5344 | return 0; |
5345 | mdelay(1000); |
5346 | } while (retrycount++ < dev->hw_attrs.max_pe_ready_count); |
5347 | return -1; |
5348 | } |
5349 | |
5350 | static inline void irdma_sc_init_hw(struct irdma_sc_dev *dev) |
5351 | { |
5352 | switch (dev->hw_attrs.uk_attrs.hw_rev) { |
5353 | case IRDMA_GEN_1: |
5354 | i40iw_init_hw(dev); |
5355 | break; |
5356 | case IRDMA_GEN_2: |
5357 | icrdma_init_hw(dev); |
5358 | break; |
5359 | } |
5360 | } |
5361 | |
5362 | /** |
5363 | * irdma_sc_dev_init - Initialize control part of device |
5364 | * @ver: version |
5365 | * @dev: Device pointer |
5366 | * @info: Device init info |
5367 | */ |
5368 | int irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev, |
5369 | struct irdma_device_init_info *info) |
5370 | { |
5371 | u32 val; |
5372 | int ret_code = 0; |
5373 | u8 db_size; |
5374 | |
5375 | INIT_LIST_HEAD(list: &dev->cqp_cmd_head); /* for CQP command backlog */ |
5376 | mutex_init(&dev->ws_mutex); |
5377 | dev->hmc_fn_id = info->hmc_fn_id; |
5378 | dev->fpm_query_buf_pa = info->fpm_query_buf_pa; |
5379 | dev->fpm_query_buf = info->fpm_query_buf; |
5380 | dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa; |
5381 | dev->fpm_commit_buf = info->fpm_commit_buf; |
5382 | dev->hw = info->hw; |
5383 | dev->hw->hw_addr = info->bar0; |
5384 | /* Setup the hardware limits, hmc may limit further */ |
5385 | dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID; |
5386 | dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES; |
5387 | dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES; |
5388 | dev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES; |
5389 | dev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES; |
5390 | dev->hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE; |
5391 | dev->hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE; |
5392 | dev->hw_attrs.uk_attrs.max_hw_wq_frags = IRDMA_MAX_WQ_FRAGMENT_COUNT; |
5393 | dev->hw_attrs.uk_attrs.max_hw_read_sges = IRDMA_MAX_SGE_RD; |
5394 | dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE; |
5395 | dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE; |
5396 | dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE; |
5397 | dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT; |
5398 | dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE; |
5399 | dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES; |
5400 | dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR); |
5401 | |
5402 | dev->hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA; |
5403 | dev->hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA; |
5404 | dev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS; |
5405 | dev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT; |
5406 | |
5407 | dev->hw_attrs.max_pe_ready_count = 14; |
5408 | dev->hw_attrs.max_done_count = IRDMA_DONE_COUNT; |
5409 | dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT; |
5410 | dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS; |
5411 | |
5412 | dev->hw_attrs.uk_attrs.hw_rev = ver; |
5413 | irdma_sc_init_hw(dev); |
5414 | |
5415 | if (irdma_wait_pe_ready(dev)) |
5416 | return -ETIMEDOUT; |
5417 | |
5418 | val = readl(addr: dev->hw_regs[IRDMA_GLPCI_LBARCTRL]); |
5419 | db_size = (u8)FIELD_GET(IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE, val); |
5420 | if (db_size != IRDMA_PE_DB_SIZE_4M && db_size != IRDMA_PE_DB_SIZE_8M) { |
5421 | ibdev_dbg(to_ibdev(dev), |
5422 | "DEV: RDMA PE doorbell is not enabled in CSR val 0x%x db_size=%d\n" , |
5423 | val, db_size); |
5424 | return -ENODEV; |
5425 | } |
5426 | dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET]; |
5427 | |
5428 | return ret_code; |
5429 | } |
5430 | |
5431 | /** |
5432 | * irdma_stat_val - Extract HW counter value from statistics buffer |
5433 | * @stats_val: pointer to statistics buffer |
5434 | * @byteoff: byte offset of counter value in the buffer (8B-aligned) |
5435 | * @bitoff: bit offset of counter value within 8B entry |
5436 | * @bitmask: maximum counter value (e.g. 0xffffff for 24-bit counter) |
5437 | */ |
5438 | static inline u64 irdma_stat_val(const u64 *stats_val, u16 byteoff, u8 bitoff, |
5439 | u64 bitmask) |
5440 | { |
5441 | u16 idx = byteoff / sizeof(*stats_val); |
5442 | |
5443 | return (stats_val[idx] >> bitoff) & bitmask; |
5444 | } |
5445 | |
5446 | /** |
5447 | * irdma_stat_delta - Calculate counter delta |
5448 | * @new_val: updated counter value |
5449 | * @old_val: last counter value |
5450 | * @max_val: maximum counter value (e.g. 0xffffff for 24-bit counter) |
5451 | */ |
5452 | static inline u64 irdma_stat_delta(u64 new_val, u64 old_val, u64 max_val) |
5453 | { |
5454 | if (new_val >= old_val) |
5455 | return new_val - old_val; |
5456 | |
5457 | /* roll-over case */ |
5458 | return max_val - old_val + new_val + 1; |
5459 | } |
5460 | |
5461 | /** |
5462 | * irdma_update_stats - Update statistics |
5463 | * @hw_stats: hw_stats instance to update |
5464 | * @gather_stats: updated stat counters |
5465 | * @last_gather_stats: last stat counters |
5466 | * @map: HW stat map (hw_stats => gather_stats) |
5467 | * @max_stat_idx: number of HW stats |
5468 | */ |
5469 | void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats, |
5470 | struct irdma_gather_stats *gather_stats, |
5471 | struct irdma_gather_stats *last_gather_stats, |
5472 | const struct irdma_hw_stat_map *map, u16 max_stat_idx) |
5473 | { |
5474 | u64 *stats_val = hw_stats->stats_val; |
5475 | u16 i; |
5476 | |
5477 | for (i = 0; i < max_stat_idx; i++) { |
5478 | u64 new_val = irdma_stat_val(stats_val: gather_stats->val, byteoff: map[i].byteoff, |
5479 | bitoff: map[i].bitoff, bitmask: map[i].bitmask); |
5480 | u64 last_val = irdma_stat_val(stats_val: last_gather_stats->val, |
5481 | byteoff: map[i].byteoff, bitoff: map[i].bitoff, |
5482 | bitmask: map[i].bitmask); |
5483 | |
5484 | stats_val[i] += |
5485 | irdma_stat_delta(new_val, old_val: last_val, max_val: map[i].bitmask); |
5486 | } |
5487 | |
5488 | memcpy(last_gather_stats, gather_stats, sizeof(*last_gather_stats)); |
5489 | } |
5490 | |