1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2018, Intel Corporation. */ |
3 | |
4 | #include "ice_common.h" |
5 | |
6 | #define ICE_CQ_INIT_REGS(qinfo, prefix) \ |
7 | do { \ |
8 | (qinfo)->sq.head = prefix##_ATQH; \ |
9 | (qinfo)->sq.tail = prefix##_ATQT; \ |
10 | (qinfo)->sq.len = prefix##_ATQLEN; \ |
11 | (qinfo)->sq.bah = prefix##_ATQBAH; \ |
12 | (qinfo)->sq.bal = prefix##_ATQBAL; \ |
13 | (qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M; \ |
14 | (qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M; \ |
15 | (qinfo)->sq.len_crit_mask = prefix##_ATQLEN_ATQCRIT_M; \ |
16 | (qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M; \ |
17 | (qinfo)->rq.head = prefix##_ARQH; \ |
18 | (qinfo)->rq.tail = prefix##_ARQT; \ |
19 | (qinfo)->rq.len = prefix##_ARQLEN; \ |
20 | (qinfo)->rq.bah = prefix##_ARQBAH; \ |
21 | (qinfo)->rq.bal = prefix##_ARQBAL; \ |
22 | (qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M; \ |
23 | (qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M; \ |
24 | (qinfo)->rq.len_crit_mask = prefix##_ARQLEN_ARQCRIT_M; \ |
25 | (qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M; \ |
26 | } while (0) |
27 | |
28 | /** |
29 | * ice_adminq_init_regs - Initialize AdminQ registers |
30 | * @hw: pointer to the hardware structure |
31 | * |
32 | * This assumes the alloc_sq and alloc_rq functions have already been called |
33 | */ |
34 | static void ice_adminq_init_regs(struct ice_hw *hw) |
35 | { |
36 | struct ice_ctl_q_info *cq = &hw->adminq; |
37 | |
38 | ICE_CQ_INIT_REGS(cq, PF_FW); |
39 | } |
40 | |
41 | /** |
42 | * ice_mailbox_init_regs - Initialize Mailbox registers |
43 | * @hw: pointer to the hardware structure |
44 | * |
45 | * This assumes the alloc_sq and alloc_rq functions have already been called |
46 | */ |
47 | static void ice_mailbox_init_regs(struct ice_hw *hw) |
48 | { |
49 | struct ice_ctl_q_info *cq = &hw->mailboxq; |
50 | |
51 | ICE_CQ_INIT_REGS(cq, PF_MBX); |
52 | } |
53 | |
54 | /** |
55 | * ice_sb_init_regs - Initialize Sideband registers |
56 | * @hw: pointer to the hardware structure |
57 | * |
58 | * This assumes the alloc_sq and alloc_rq functions have already been called |
59 | */ |
60 | static void ice_sb_init_regs(struct ice_hw *hw) |
61 | { |
62 | struct ice_ctl_q_info *cq = &hw->sbq; |
63 | |
64 | ICE_CQ_INIT_REGS(cq, PF_SB); |
65 | } |
66 | |
67 | /** |
68 | * ice_check_sq_alive |
69 | * @hw: pointer to the HW struct |
70 | * @cq: pointer to the specific Control queue |
71 | * |
72 | * Returns true if Queue is enabled else false. |
73 | */ |
74 | bool ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
75 | { |
76 | /* check both queue-length and queue-enable fields */ |
77 | if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask) |
78 | return (rd32(hw, cq->sq.len) & (cq->sq.len_mask | |
79 | cq->sq.len_ena_mask)) == |
80 | (cq->num_sq_entries | cq->sq.len_ena_mask); |
81 | |
82 | return false; |
83 | } |
84 | |
85 | /** |
86 | * ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings |
87 | * @hw: pointer to the hardware structure |
88 | * @cq: pointer to the specific Control queue |
89 | */ |
90 | static int |
91 | ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
92 | { |
93 | size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc); |
94 | |
95 | cq->sq.desc_buf.va = dmam_alloc_coherent(dev: ice_hw_to_dev(hw), size, |
96 | dma_handle: &cq->sq.desc_buf.pa, |
97 | GFP_KERNEL | __GFP_ZERO); |
98 | if (!cq->sq.desc_buf.va) |
99 | return -ENOMEM; |
100 | cq->sq.desc_buf.size = size; |
101 | |
102 | cq->sq.cmd_buf = devm_kcalloc(dev: ice_hw_to_dev(hw), n: cq->num_sq_entries, |
103 | size: sizeof(struct ice_sq_cd), GFP_KERNEL); |
104 | if (!cq->sq.cmd_buf) { |
105 | dmam_free_coherent(dev: ice_hw_to_dev(hw), size: cq->sq.desc_buf.size, |
106 | vaddr: cq->sq.desc_buf.va, dma_handle: cq->sq.desc_buf.pa); |
107 | cq->sq.desc_buf.va = NULL; |
108 | cq->sq.desc_buf.pa = 0; |
109 | cq->sq.desc_buf.size = 0; |
110 | return -ENOMEM; |
111 | } |
112 | |
113 | return 0; |
114 | } |
115 | |
116 | /** |
117 | * ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings |
118 | * @hw: pointer to the hardware structure |
119 | * @cq: pointer to the specific Control queue |
120 | */ |
121 | static int |
122 | ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
123 | { |
124 | size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc); |
125 | |
126 | cq->rq.desc_buf.va = dmam_alloc_coherent(dev: ice_hw_to_dev(hw), size, |
127 | dma_handle: &cq->rq.desc_buf.pa, |
128 | GFP_KERNEL | __GFP_ZERO); |
129 | if (!cq->rq.desc_buf.va) |
130 | return -ENOMEM; |
131 | cq->rq.desc_buf.size = size; |
132 | return 0; |
133 | } |
134 | |
135 | /** |
136 | * ice_free_cq_ring - Free control queue ring |
137 | * @hw: pointer to the hardware structure |
138 | * @ring: pointer to the specific control queue ring |
139 | * |
140 | * This assumes the posted buffers have already been cleaned |
141 | * and de-allocated |
142 | */ |
143 | static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring) |
144 | { |
145 | dmam_free_coherent(dev: ice_hw_to_dev(hw), size: ring->desc_buf.size, |
146 | vaddr: ring->desc_buf.va, dma_handle: ring->desc_buf.pa); |
147 | ring->desc_buf.va = NULL; |
148 | ring->desc_buf.pa = 0; |
149 | ring->desc_buf.size = 0; |
150 | } |
151 | |
152 | /** |
153 | * ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ |
154 | * @hw: pointer to the hardware structure |
155 | * @cq: pointer to the specific Control queue |
156 | */ |
157 | static int |
158 | ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
159 | { |
160 | int i; |
161 | |
162 | /* We'll be allocating the buffer info memory first, then we can |
163 | * allocate the mapped buffers for the event processing |
164 | */ |
165 | cq->rq.dma_head = devm_kcalloc(dev: ice_hw_to_dev(hw), n: cq->num_rq_entries, |
166 | size: sizeof(cq->rq.desc_buf), GFP_KERNEL); |
167 | if (!cq->rq.dma_head) |
168 | return -ENOMEM; |
169 | cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head; |
170 | |
171 | /* allocate the mapped buffers */ |
172 | for (i = 0; i < cq->num_rq_entries; i++) { |
173 | struct ice_aq_desc *desc; |
174 | struct ice_dma_mem *bi; |
175 | |
176 | bi = &cq->rq.r.rq_bi[i]; |
177 | bi->va = dmam_alloc_coherent(dev: ice_hw_to_dev(hw), |
178 | size: cq->rq_buf_size, dma_handle: &bi->pa, |
179 | GFP_KERNEL | __GFP_ZERO); |
180 | if (!bi->va) |
181 | goto unwind_alloc_rq_bufs; |
182 | bi->size = cq->rq_buf_size; |
183 | |
184 | /* now configure the descriptors for use */ |
185 | desc = ICE_CTL_Q_DESC(cq->rq, i); |
186 | |
187 | desc->flags = cpu_to_le16(ICE_AQ_FLAG_BUF); |
188 | if (cq->rq_buf_size > ICE_AQ_LG_BUF) |
189 | desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB); |
190 | desc->opcode = 0; |
191 | /* This is in accordance with Admin queue design, there is no |
192 | * register for buffer size configuration |
193 | */ |
194 | desc->datalen = cpu_to_le16(bi->size); |
195 | desc->retval = 0; |
196 | desc->cookie_high = 0; |
197 | desc->cookie_low = 0; |
198 | desc->params.generic.addr_high = |
199 | cpu_to_le32(upper_32_bits(bi->pa)); |
200 | desc->params.generic.addr_low = |
201 | cpu_to_le32(lower_32_bits(bi->pa)); |
202 | desc->params.generic.param0 = 0; |
203 | desc->params.generic.param1 = 0; |
204 | } |
205 | return 0; |
206 | |
207 | unwind_alloc_rq_bufs: |
208 | /* don't try to free the one that failed... */ |
209 | i--; |
210 | for (; i >= 0; i--) { |
211 | dmam_free_coherent(dev: ice_hw_to_dev(hw), size: cq->rq.r.rq_bi[i].size, |
212 | vaddr: cq->rq.r.rq_bi[i].va, dma_handle: cq->rq.r.rq_bi[i].pa); |
213 | cq->rq.r.rq_bi[i].va = NULL; |
214 | cq->rq.r.rq_bi[i].pa = 0; |
215 | cq->rq.r.rq_bi[i].size = 0; |
216 | } |
217 | cq->rq.r.rq_bi = NULL; |
218 | devm_kfree(dev: ice_hw_to_dev(hw), p: cq->rq.dma_head); |
219 | cq->rq.dma_head = NULL; |
220 | |
221 | return -ENOMEM; |
222 | } |
223 | |
224 | /** |
225 | * ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ |
226 | * @hw: pointer to the hardware structure |
227 | * @cq: pointer to the specific Control queue |
228 | */ |
229 | static int |
230 | ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
231 | { |
232 | int i; |
233 | |
234 | /* No mapped memory needed yet, just the buffer info structures */ |
235 | cq->sq.dma_head = devm_kcalloc(dev: ice_hw_to_dev(hw), n: cq->num_sq_entries, |
236 | size: sizeof(cq->sq.desc_buf), GFP_KERNEL); |
237 | if (!cq->sq.dma_head) |
238 | return -ENOMEM; |
239 | cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head; |
240 | |
241 | /* allocate the mapped buffers */ |
242 | for (i = 0; i < cq->num_sq_entries; i++) { |
243 | struct ice_dma_mem *bi; |
244 | |
245 | bi = &cq->sq.r.sq_bi[i]; |
246 | bi->va = dmam_alloc_coherent(dev: ice_hw_to_dev(hw), |
247 | size: cq->sq_buf_size, dma_handle: &bi->pa, |
248 | GFP_KERNEL | __GFP_ZERO); |
249 | if (!bi->va) |
250 | goto unwind_alloc_sq_bufs; |
251 | bi->size = cq->sq_buf_size; |
252 | } |
253 | return 0; |
254 | |
255 | unwind_alloc_sq_bufs: |
256 | /* don't try to free the one that failed... */ |
257 | i--; |
258 | for (; i >= 0; i--) { |
259 | dmam_free_coherent(dev: ice_hw_to_dev(hw), size: cq->sq.r.sq_bi[i].size, |
260 | vaddr: cq->sq.r.sq_bi[i].va, dma_handle: cq->sq.r.sq_bi[i].pa); |
261 | cq->sq.r.sq_bi[i].va = NULL; |
262 | cq->sq.r.sq_bi[i].pa = 0; |
263 | cq->sq.r.sq_bi[i].size = 0; |
264 | } |
265 | cq->sq.r.sq_bi = NULL; |
266 | devm_kfree(dev: ice_hw_to_dev(hw), p: cq->sq.dma_head); |
267 | cq->sq.dma_head = NULL; |
268 | |
269 | return -ENOMEM; |
270 | } |
271 | |
272 | static int |
273 | ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries) |
274 | { |
275 | /* Clear Head and Tail */ |
276 | wr32(hw, ring->head, 0); |
277 | wr32(hw, ring->tail, 0); |
278 | |
279 | /* set starting point */ |
280 | wr32(hw, ring->len, (num_entries | ring->len_ena_mask)); |
281 | wr32(hw, ring->bal, lower_32_bits(ring->desc_buf.pa)); |
282 | wr32(hw, ring->bah, upper_32_bits(ring->desc_buf.pa)); |
283 | |
284 | /* Check one register to verify that config was applied */ |
285 | if (rd32(hw, ring->bal) != lower_32_bits(ring->desc_buf.pa)) |
286 | return -EIO; |
287 | |
288 | return 0; |
289 | } |
290 | |
291 | /** |
292 | * ice_cfg_sq_regs - configure Control ATQ registers |
293 | * @hw: pointer to the hardware structure |
294 | * @cq: pointer to the specific Control queue |
295 | * |
296 | * Configure base address and length registers for the transmit queue |
297 | */ |
298 | static int ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
299 | { |
300 | return ice_cfg_cq_regs(hw, ring: &cq->sq, num_entries: cq->num_sq_entries); |
301 | } |
302 | |
303 | /** |
304 | * ice_cfg_rq_regs - configure Control ARQ register |
305 | * @hw: pointer to the hardware structure |
306 | * @cq: pointer to the specific Control queue |
307 | * |
308 | * Configure base address and length registers for the receive (event queue) |
309 | */ |
310 | static int ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
311 | { |
312 | int status; |
313 | |
314 | status = ice_cfg_cq_regs(hw, ring: &cq->rq, num_entries: cq->num_rq_entries); |
315 | if (status) |
316 | return status; |
317 | |
318 | /* Update tail in the HW to post pre-allocated buffers */ |
319 | wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1)); |
320 | |
321 | return 0; |
322 | } |
323 | |
324 | #define ICE_FREE_CQ_BUFS(hw, qi, ring) \ |
325 | do { \ |
326 | /* free descriptors */ \ |
327 | if ((qi)->ring.r.ring##_bi) { \ |
328 | int i; \ |
329 | \ |
330 | for (i = 0; i < (qi)->num_##ring##_entries; i++) \ |
331 | if ((qi)->ring.r.ring##_bi[i].pa) { \ |
332 | dmam_free_coherent(ice_hw_to_dev(hw), \ |
333 | (qi)->ring.r.ring##_bi[i].size, \ |
334 | (qi)->ring.r.ring##_bi[i].va, \ |
335 | (qi)->ring.r.ring##_bi[i].pa); \ |
336 | (qi)->ring.r.ring##_bi[i].va = NULL;\ |
337 | (qi)->ring.r.ring##_bi[i].pa = 0;\ |
338 | (qi)->ring.r.ring##_bi[i].size = 0;\ |
339 | } \ |
340 | } \ |
341 | /* free the buffer info list */ \ |
342 | devm_kfree(ice_hw_to_dev(hw), (qi)->ring.cmd_buf); \ |
343 | /* free DMA head */ \ |
344 | devm_kfree(ice_hw_to_dev(hw), (qi)->ring.dma_head); \ |
345 | } while (0) |
346 | |
347 | /** |
348 | * ice_init_sq - main initialization routine for Control ATQ |
349 | * @hw: pointer to the hardware structure |
350 | * @cq: pointer to the specific Control queue |
351 | * |
352 | * This is the main initialization routine for the Control Send Queue |
353 | * Prior to calling this function, the driver *MUST* set the following fields |
354 | * in the cq->structure: |
355 | * - cq->num_sq_entries |
356 | * - cq->sq_buf_size |
357 | * |
358 | * Do *NOT* hold the lock when calling this as the memory allocation routines |
359 | * called are not going to be atomic context safe |
360 | */ |
361 | static int ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
362 | { |
363 | int ret_code; |
364 | |
365 | if (cq->sq.count > 0) { |
366 | /* queue already initialized */ |
367 | ret_code = -EBUSY; |
368 | goto init_ctrlq_exit; |
369 | } |
370 | |
371 | /* verify input for valid configuration */ |
372 | if (!cq->num_sq_entries || !cq->sq_buf_size) { |
373 | ret_code = -EIO; |
374 | goto init_ctrlq_exit; |
375 | } |
376 | |
377 | cq->sq.next_to_use = 0; |
378 | cq->sq.next_to_clean = 0; |
379 | |
380 | /* allocate the ring memory */ |
381 | ret_code = ice_alloc_ctrlq_sq_ring(hw, cq); |
382 | if (ret_code) |
383 | goto init_ctrlq_exit; |
384 | |
385 | /* allocate buffers in the rings */ |
386 | ret_code = ice_alloc_sq_bufs(hw, cq); |
387 | if (ret_code) |
388 | goto init_ctrlq_free_rings; |
389 | |
390 | /* initialize base registers */ |
391 | ret_code = ice_cfg_sq_regs(hw, cq); |
392 | if (ret_code) |
393 | goto init_ctrlq_free_rings; |
394 | |
395 | /* success! */ |
396 | cq->sq.count = cq->num_sq_entries; |
397 | goto init_ctrlq_exit; |
398 | |
399 | init_ctrlq_free_rings: |
400 | ICE_FREE_CQ_BUFS(hw, cq, sq); |
401 | ice_free_cq_ring(hw, ring: &cq->sq); |
402 | |
403 | init_ctrlq_exit: |
404 | return ret_code; |
405 | } |
406 | |
407 | /** |
408 | * ice_init_rq - initialize ARQ |
409 | * @hw: pointer to the hardware structure |
410 | * @cq: pointer to the specific Control queue |
411 | * |
412 | * The main initialization routine for the Admin Receive (Event) Queue. |
413 | * Prior to calling this function, the driver *MUST* set the following fields |
414 | * in the cq->structure: |
415 | * - cq->num_rq_entries |
416 | * - cq->rq_buf_size |
417 | * |
418 | * Do *NOT* hold the lock when calling this as the memory allocation routines |
419 | * called are not going to be atomic context safe |
420 | */ |
421 | static int ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
422 | { |
423 | int ret_code; |
424 | |
425 | if (cq->rq.count > 0) { |
426 | /* queue already initialized */ |
427 | ret_code = -EBUSY; |
428 | goto init_ctrlq_exit; |
429 | } |
430 | |
431 | /* verify input for valid configuration */ |
432 | if (!cq->num_rq_entries || !cq->rq_buf_size) { |
433 | ret_code = -EIO; |
434 | goto init_ctrlq_exit; |
435 | } |
436 | |
437 | cq->rq.next_to_use = 0; |
438 | cq->rq.next_to_clean = 0; |
439 | |
440 | /* allocate the ring memory */ |
441 | ret_code = ice_alloc_ctrlq_rq_ring(hw, cq); |
442 | if (ret_code) |
443 | goto init_ctrlq_exit; |
444 | |
445 | /* allocate buffers in the rings */ |
446 | ret_code = ice_alloc_rq_bufs(hw, cq); |
447 | if (ret_code) |
448 | goto init_ctrlq_free_rings; |
449 | |
450 | /* initialize base registers */ |
451 | ret_code = ice_cfg_rq_regs(hw, cq); |
452 | if (ret_code) |
453 | goto init_ctrlq_free_rings; |
454 | |
455 | /* success! */ |
456 | cq->rq.count = cq->num_rq_entries; |
457 | goto init_ctrlq_exit; |
458 | |
459 | init_ctrlq_free_rings: |
460 | ICE_FREE_CQ_BUFS(hw, cq, rq); |
461 | ice_free_cq_ring(hw, ring: &cq->rq); |
462 | |
463 | init_ctrlq_exit: |
464 | return ret_code; |
465 | } |
466 | |
467 | /** |
468 | * ice_shutdown_sq - shutdown the Control ATQ |
469 | * @hw: pointer to the hardware structure |
470 | * @cq: pointer to the specific Control queue |
471 | * |
472 | * The main shutdown routine for the Control Transmit Queue |
473 | */ |
474 | static int ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
475 | { |
476 | int ret_code = 0; |
477 | |
478 | mutex_lock(&cq->sq_lock); |
479 | |
480 | if (!cq->sq.count) { |
481 | ret_code = -EBUSY; |
482 | goto shutdown_sq_out; |
483 | } |
484 | |
485 | /* Stop firmware AdminQ processing */ |
486 | wr32(hw, cq->sq.head, 0); |
487 | wr32(hw, cq->sq.tail, 0); |
488 | wr32(hw, cq->sq.len, 0); |
489 | wr32(hw, cq->sq.bal, 0); |
490 | wr32(hw, cq->sq.bah, 0); |
491 | |
492 | cq->sq.count = 0; /* to indicate uninitialized queue */ |
493 | |
494 | /* free ring buffers and the ring itself */ |
495 | ICE_FREE_CQ_BUFS(hw, cq, sq); |
496 | ice_free_cq_ring(hw, ring: &cq->sq); |
497 | |
498 | shutdown_sq_out: |
499 | mutex_unlock(lock: &cq->sq_lock); |
500 | return ret_code; |
501 | } |
502 | |
503 | /** |
504 | * ice_aq_ver_check - Check the reported AQ API version. |
505 | * @hw: pointer to the hardware structure |
506 | * |
507 | * Checks if the driver should load on a given AQ API version. |
508 | * |
509 | * Return: 'true' iff the driver should attempt to load. 'false' otherwise. |
510 | */ |
511 | static bool ice_aq_ver_check(struct ice_hw *hw) |
512 | { |
513 | if (hw->api_maj_ver > EXP_FW_API_VER_MAJOR) { |
514 | /* Major API version is newer than expected, don't load */ |
515 | dev_warn(ice_hw_to_dev(hw), |
516 | "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n" ); |
517 | return false; |
518 | } else if (hw->api_maj_ver == EXP_FW_API_VER_MAJOR) { |
519 | if (hw->api_min_ver > (EXP_FW_API_VER_MINOR + 2)) |
520 | dev_info(ice_hw_to_dev(hw), |
521 | "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n" ); |
522 | else if ((hw->api_min_ver + 2) < EXP_FW_API_VER_MINOR) |
523 | dev_info(ice_hw_to_dev(hw), |
524 | "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n" ); |
525 | } else { |
526 | /* Major API version is older than expected, log a warning */ |
527 | dev_info(ice_hw_to_dev(hw), |
528 | "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n" ); |
529 | } |
530 | return true; |
531 | } |
532 | |
533 | /** |
534 | * ice_shutdown_rq - shutdown Control ARQ |
535 | * @hw: pointer to the hardware structure |
536 | * @cq: pointer to the specific Control queue |
537 | * |
538 | * The main shutdown routine for the Control Receive Queue |
539 | */ |
540 | static int ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
541 | { |
542 | int ret_code = 0; |
543 | |
544 | mutex_lock(&cq->rq_lock); |
545 | |
546 | if (!cq->rq.count) { |
547 | ret_code = -EBUSY; |
548 | goto shutdown_rq_out; |
549 | } |
550 | |
551 | /* Stop Control Queue processing */ |
552 | wr32(hw, cq->rq.head, 0); |
553 | wr32(hw, cq->rq.tail, 0); |
554 | wr32(hw, cq->rq.len, 0); |
555 | wr32(hw, cq->rq.bal, 0); |
556 | wr32(hw, cq->rq.bah, 0); |
557 | |
558 | /* set rq.count to 0 to indicate uninitialized queue */ |
559 | cq->rq.count = 0; |
560 | |
561 | /* free ring buffers and the ring itself */ |
562 | ICE_FREE_CQ_BUFS(hw, cq, rq); |
563 | ice_free_cq_ring(hw, ring: &cq->rq); |
564 | |
565 | shutdown_rq_out: |
566 | mutex_unlock(lock: &cq->rq_lock); |
567 | return ret_code; |
568 | } |
569 | |
570 | /** |
571 | * ice_init_check_adminq - Check version for Admin Queue to know if its alive |
572 | * @hw: pointer to the hardware structure |
573 | */ |
574 | static int ice_init_check_adminq(struct ice_hw *hw) |
575 | { |
576 | struct ice_ctl_q_info *cq = &hw->adminq; |
577 | int status; |
578 | |
579 | status = ice_aq_get_fw_ver(hw, NULL); |
580 | if (status) |
581 | goto init_ctrlq_free_rq; |
582 | |
583 | if (!ice_aq_ver_check(hw)) { |
584 | status = -EIO; |
585 | goto init_ctrlq_free_rq; |
586 | } |
587 | |
588 | return 0; |
589 | |
590 | init_ctrlq_free_rq: |
591 | ice_shutdown_rq(hw, cq); |
592 | ice_shutdown_sq(hw, cq); |
593 | return status; |
594 | } |
595 | |
596 | /** |
597 | * ice_init_ctrlq - main initialization routine for any control Queue |
598 | * @hw: pointer to the hardware structure |
599 | * @q_type: specific Control queue type |
600 | * |
601 | * Prior to calling this function, the driver *MUST* set the following fields |
602 | * in the cq->structure: |
603 | * - cq->num_sq_entries |
604 | * - cq->num_rq_entries |
605 | * - cq->rq_buf_size |
606 | * - cq->sq_buf_size |
607 | * |
608 | * NOTE: this function does not initialize the controlq locks |
609 | */ |
610 | static int ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type) |
611 | { |
612 | struct ice_ctl_q_info *cq; |
613 | int ret_code; |
614 | |
615 | switch (q_type) { |
616 | case ICE_CTL_Q_ADMIN: |
617 | ice_adminq_init_regs(hw); |
618 | cq = &hw->adminq; |
619 | break; |
620 | case ICE_CTL_Q_SB: |
621 | ice_sb_init_regs(hw); |
622 | cq = &hw->sbq; |
623 | break; |
624 | case ICE_CTL_Q_MAILBOX: |
625 | ice_mailbox_init_regs(hw); |
626 | cq = &hw->mailboxq; |
627 | break; |
628 | default: |
629 | return -EINVAL; |
630 | } |
631 | cq->qtype = q_type; |
632 | |
633 | /* verify input for valid configuration */ |
634 | if (!cq->num_rq_entries || !cq->num_sq_entries || |
635 | !cq->rq_buf_size || !cq->sq_buf_size) { |
636 | return -EIO; |
637 | } |
638 | |
639 | /* allocate the ATQ */ |
640 | ret_code = ice_init_sq(hw, cq); |
641 | if (ret_code) |
642 | return ret_code; |
643 | |
644 | /* allocate the ARQ */ |
645 | ret_code = ice_init_rq(hw, cq); |
646 | if (ret_code) |
647 | goto init_ctrlq_free_sq; |
648 | |
649 | /* success! */ |
650 | return 0; |
651 | |
652 | init_ctrlq_free_sq: |
653 | ice_shutdown_sq(hw, cq); |
654 | return ret_code; |
655 | } |
656 | |
657 | /** |
658 | * ice_is_sbq_supported - is the sideband queue supported |
659 | * @hw: pointer to the hardware structure |
660 | * |
661 | * Returns true if the sideband control queue interface is |
662 | * supported for the device, false otherwise |
663 | */ |
664 | bool ice_is_sbq_supported(struct ice_hw *hw) |
665 | { |
666 | /* The device sideband queue is only supported on devices with the |
667 | * generic MAC type. |
668 | */ |
669 | return hw->mac_type == ICE_MAC_GENERIC; |
670 | } |
671 | |
672 | /** |
673 | * ice_get_sbq - returns the right control queue to use for sideband |
674 | * @hw: pointer to the hardware structure |
675 | */ |
676 | struct ice_ctl_q_info *ice_get_sbq(struct ice_hw *hw) |
677 | { |
678 | if (ice_is_sbq_supported(hw)) |
679 | return &hw->sbq; |
680 | return &hw->adminq; |
681 | } |
682 | |
683 | /** |
684 | * ice_shutdown_ctrlq - shutdown routine for any control queue |
685 | * @hw: pointer to the hardware structure |
686 | * @q_type: specific Control queue type |
687 | * |
688 | * NOTE: this function does not destroy the control queue locks. |
689 | */ |
690 | static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type) |
691 | { |
692 | struct ice_ctl_q_info *cq; |
693 | |
694 | switch (q_type) { |
695 | case ICE_CTL_Q_ADMIN: |
696 | cq = &hw->adminq; |
697 | if (ice_check_sq_alive(hw, cq)) |
698 | ice_aq_q_shutdown(hw, unloading: true); |
699 | break; |
700 | case ICE_CTL_Q_SB: |
701 | cq = &hw->sbq; |
702 | break; |
703 | case ICE_CTL_Q_MAILBOX: |
704 | cq = &hw->mailboxq; |
705 | break; |
706 | default: |
707 | return; |
708 | } |
709 | |
710 | ice_shutdown_sq(hw, cq); |
711 | ice_shutdown_rq(hw, cq); |
712 | } |
713 | |
714 | /** |
715 | * ice_shutdown_all_ctrlq - shutdown routine for all control queues |
716 | * @hw: pointer to the hardware structure |
717 | * |
718 | * NOTE: this function does not destroy the control queue locks. The driver |
719 | * may call this at runtime to shutdown and later restart control queues, such |
720 | * as in response to a reset event. |
721 | */ |
722 | void ice_shutdown_all_ctrlq(struct ice_hw *hw) |
723 | { |
724 | /* Shutdown FW admin queue */ |
725 | ice_shutdown_ctrlq(hw, q_type: ICE_CTL_Q_ADMIN); |
726 | /* Shutdown PHY Sideband */ |
727 | if (ice_is_sbq_supported(hw)) |
728 | ice_shutdown_ctrlq(hw, q_type: ICE_CTL_Q_SB); |
729 | /* Shutdown PF-VF Mailbox */ |
730 | ice_shutdown_ctrlq(hw, q_type: ICE_CTL_Q_MAILBOX); |
731 | } |
732 | |
733 | /** |
734 | * ice_init_all_ctrlq - main initialization routine for all control queues |
735 | * @hw: pointer to the hardware structure |
736 | * |
737 | * Prior to calling this function, the driver MUST* set the following fields |
738 | * in the cq->structure for all control queues: |
739 | * - cq->num_sq_entries |
740 | * - cq->num_rq_entries |
741 | * - cq->rq_buf_size |
742 | * - cq->sq_buf_size |
743 | * |
744 | * NOTE: this function does not initialize the controlq locks. |
745 | */ |
746 | int ice_init_all_ctrlq(struct ice_hw *hw) |
747 | { |
748 | u32 retry = 0; |
749 | int status; |
750 | |
751 | /* Init FW admin queue */ |
752 | do { |
753 | status = ice_init_ctrlq(hw, q_type: ICE_CTL_Q_ADMIN); |
754 | if (status) |
755 | return status; |
756 | |
757 | status = ice_init_check_adminq(hw); |
758 | if (status != -EIO) |
759 | break; |
760 | |
761 | ice_debug(hw, ICE_DBG_AQ_MSG, "Retry Admin Queue init due to FW critical error\n" ); |
762 | ice_shutdown_ctrlq(hw, q_type: ICE_CTL_Q_ADMIN); |
763 | msleep(ICE_CTL_Q_ADMIN_INIT_MSEC); |
764 | } while (retry++ < ICE_CTL_Q_ADMIN_INIT_TIMEOUT); |
765 | |
766 | if (status) |
767 | return status; |
768 | /* sideband control queue (SBQ) interface is not supported on some |
769 | * devices. Initialize if supported, else fallback to the admin queue |
770 | * interface |
771 | */ |
772 | if (ice_is_sbq_supported(hw)) { |
773 | status = ice_init_ctrlq(hw, q_type: ICE_CTL_Q_SB); |
774 | if (status) |
775 | return status; |
776 | } |
777 | /* Init Mailbox queue */ |
778 | return ice_init_ctrlq(hw, q_type: ICE_CTL_Q_MAILBOX); |
779 | } |
780 | |
781 | /** |
782 | * ice_init_ctrlq_locks - Initialize locks for a control queue |
783 | * @cq: pointer to the control queue |
784 | * |
785 | * Initializes the send and receive queue locks for a given control queue. |
786 | */ |
787 | static void ice_init_ctrlq_locks(struct ice_ctl_q_info *cq) |
788 | { |
789 | mutex_init(&cq->sq_lock); |
790 | mutex_init(&cq->rq_lock); |
791 | } |
792 | |
793 | /** |
794 | * ice_create_all_ctrlq - main initialization routine for all control queues |
795 | * @hw: pointer to the hardware structure |
796 | * |
797 | * Prior to calling this function, the driver *MUST* set the following fields |
798 | * in the cq->structure for all control queues: |
799 | * - cq->num_sq_entries |
800 | * - cq->num_rq_entries |
801 | * - cq->rq_buf_size |
802 | * - cq->sq_buf_size |
803 | * |
804 | * This function creates all the control queue locks and then calls |
805 | * ice_init_all_ctrlq. It should be called once during driver load. If the |
806 | * driver needs to re-initialize control queues at run time it should call |
807 | * ice_init_all_ctrlq instead. |
808 | */ |
809 | int ice_create_all_ctrlq(struct ice_hw *hw) |
810 | { |
811 | ice_init_ctrlq_locks(cq: &hw->adminq); |
812 | if (ice_is_sbq_supported(hw)) |
813 | ice_init_ctrlq_locks(cq: &hw->sbq); |
814 | ice_init_ctrlq_locks(cq: &hw->mailboxq); |
815 | |
816 | return ice_init_all_ctrlq(hw); |
817 | } |
818 | |
819 | /** |
820 | * ice_destroy_ctrlq_locks - Destroy locks for a control queue |
821 | * @cq: pointer to the control queue |
822 | * |
823 | * Destroys the send and receive queue locks for a given control queue. |
824 | */ |
825 | static void ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq) |
826 | { |
827 | mutex_destroy(lock: &cq->sq_lock); |
828 | mutex_destroy(lock: &cq->rq_lock); |
829 | } |
830 | |
831 | /** |
832 | * ice_destroy_all_ctrlq - exit routine for all control queues |
833 | * @hw: pointer to the hardware structure |
834 | * |
835 | * This function shuts down all the control queues and then destroys the |
836 | * control queue locks. It should be called once during driver unload. The |
837 | * driver should call ice_shutdown_all_ctrlq if it needs to shut down and |
838 | * reinitialize control queues, such as in response to a reset event. |
839 | */ |
840 | void ice_destroy_all_ctrlq(struct ice_hw *hw) |
841 | { |
842 | /* shut down all the control queues first */ |
843 | ice_shutdown_all_ctrlq(hw); |
844 | |
845 | ice_destroy_ctrlq_locks(cq: &hw->adminq); |
846 | if (ice_is_sbq_supported(hw)) |
847 | ice_destroy_ctrlq_locks(cq: &hw->sbq); |
848 | ice_destroy_ctrlq_locks(cq: &hw->mailboxq); |
849 | } |
850 | |
851 | /** |
852 | * ice_clean_sq - cleans Admin send queue (ATQ) |
853 | * @hw: pointer to the hardware structure |
854 | * @cq: pointer to the specific Control queue |
855 | * |
856 | * returns the number of free desc |
857 | */ |
858 | static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
859 | { |
860 | struct ice_ctl_q_ring *sq = &cq->sq; |
861 | u16 ntc = sq->next_to_clean; |
862 | struct ice_sq_cd *details; |
863 | struct ice_aq_desc *desc; |
864 | |
865 | desc = ICE_CTL_Q_DESC(*sq, ntc); |
866 | details = ICE_CTL_Q_DETAILS(*sq, ntc); |
867 | |
868 | while (rd32(hw, cq->sq.head) != ntc) { |
869 | ice_debug(hw, ICE_DBG_AQ_MSG, "ntc %d head %d.\n" , ntc, rd32(hw, cq->sq.head)); |
870 | memset(desc, 0, sizeof(*desc)); |
871 | memset(details, 0, sizeof(*details)); |
872 | ntc++; |
873 | if (ntc == sq->count) |
874 | ntc = 0; |
875 | desc = ICE_CTL_Q_DESC(*sq, ntc); |
876 | details = ICE_CTL_Q_DETAILS(*sq, ntc); |
877 | } |
878 | |
879 | sq->next_to_clean = ntc; |
880 | |
881 | return ICE_CTL_Q_DESC_UNUSED(sq); |
882 | } |
883 | |
884 | /** |
885 | * ice_debug_cq |
886 | * @hw: pointer to the hardware structure |
887 | * @desc: pointer to control queue descriptor |
888 | * @buf: pointer to command buffer |
889 | * @buf_len: max length of buf |
890 | * |
891 | * Dumps debug log about control command with descriptor contents. |
892 | */ |
893 | static void ice_debug_cq(struct ice_hw *hw, void *desc, void *buf, u16 buf_len) |
894 | { |
895 | struct ice_aq_desc *cq_desc = desc; |
896 | u16 len; |
897 | |
898 | if (!IS_ENABLED(CONFIG_DYNAMIC_DEBUG) && |
899 | !((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask)) |
900 | return; |
901 | |
902 | if (!desc) |
903 | return; |
904 | |
905 | len = le16_to_cpu(cq_desc->datalen); |
906 | |
907 | ice_debug(hw, ICE_DBG_AQ_DESC, "CQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n" , |
908 | le16_to_cpu(cq_desc->opcode), |
909 | le16_to_cpu(cq_desc->flags), |
910 | le16_to_cpu(cq_desc->datalen), le16_to_cpu(cq_desc->retval)); |
911 | ice_debug(hw, ICE_DBG_AQ_DESC, "\tcookie (h,l) 0x%08X 0x%08X\n" , |
912 | le32_to_cpu(cq_desc->cookie_high), |
913 | le32_to_cpu(cq_desc->cookie_low)); |
914 | ice_debug(hw, ICE_DBG_AQ_DESC, "\tparam (0,1) 0x%08X 0x%08X\n" , |
915 | le32_to_cpu(cq_desc->params.generic.param0), |
916 | le32_to_cpu(cq_desc->params.generic.param1)); |
917 | ice_debug(hw, ICE_DBG_AQ_DESC, "\taddr (h,l) 0x%08X 0x%08X\n" , |
918 | le32_to_cpu(cq_desc->params.generic.addr_high), |
919 | le32_to_cpu(cq_desc->params.generic.addr_low)); |
920 | if (buf && cq_desc->datalen != 0) { |
921 | ice_debug(hw, ICE_DBG_AQ_DESC_BUF, "Buffer:\n" ); |
922 | if (buf_len < len) |
923 | len = buf_len; |
924 | |
925 | ice_debug_array(hw, ICE_DBG_AQ_DESC_BUF, 16, 1, buf, len); |
926 | } |
927 | } |
928 | |
929 | /** |
930 | * ice_sq_done - check if FW has processed the Admin Send Queue (ATQ) |
931 | * @hw: pointer to the HW struct |
932 | * @cq: pointer to the specific Control queue |
933 | * |
934 | * Returns true if the firmware has processed all descriptors on the |
935 | * admin send queue. Returns false if there are still requests pending. |
936 | */ |
937 | static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq) |
938 | { |
939 | /* AQ designers suggest use of head for better |
940 | * timing reliability than DD bit |
941 | */ |
942 | return rd32(hw, cq->sq.head) == cq->sq.next_to_use; |
943 | } |
944 | |
945 | /** |
946 | * ice_sq_send_cmd - send command to Control Queue (ATQ) |
947 | * @hw: pointer to the HW struct |
948 | * @cq: pointer to the specific Control queue |
949 | * @desc: prefilled descriptor describing the command |
950 | * @buf: buffer to use for indirect commands (or NULL for direct commands) |
951 | * @buf_size: size of buffer for indirect commands (or 0 for direct commands) |
952 | * @cd: pointer to command details structure |
953 | * |
954 | * This is the main send command routine for the ATQ. It runs the queue, |
955 | * cleans the queue, etc. |
956 | */ |
957 | int |
958 | ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, |
959 | struct ice_aq_desc *desc, void *buf, u16 buf_size, |
960 | struct ice_sq_cd *cd) |
961 | { |
962 | struct ice_dma_mem *dma_buf = NULL; |
963 | struct ice_aq_desc *desc_on_ring; |
964 | bool cmd_completed = false; |
965 | struct ice_sq_cd *details; |
966 | unsigned long timeout; |
967 | int status = 0; |
968 | u16 retval = 0; |
969 | u32 val = 0; |
970 | |
971 | /* if reset is in progress return a soft error */ |
972 | if (hw->reset_ongoing) |
973 | return -EBUSY; |
974 | mutex_lock(&cq->sq_lock); |
975 | |
976 | cq->sq_last_status = ICE_AQ_RC_OK; |
977 | |
978 | if (!cq->sq.count) { |
979 | ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send queue not initialized.\n" ); |
980 | status = -EIO; |
981 | goto sq_send_command_error; |
982 | } |
983 | |
984 | if ((buf && !buf_size) || (!buf && buf_size)) { |
985 | status = -EINVAL; |
986 | goto sq_send_command_error; |
987 | } |
988 | |
989 | if (buf) { |
990 | if (buf_size > cq->sq_buf_size) { |
991 | ice_debug(hw, ICE_DBG_AQ_MSG, "Invalid buffer size for Control Send queue: %d.\n" , |
992 | buf_size); |
993 | status = -EINVAL; |
994 | goto sq_send_command_error; |
995 | } |
996 | |
997 | desc->flags |= cpu_to_le16(ICE_AQ_FLAG_BUF); |
998 | if (buf_size > ICE_AQ_LG_BUF) |
999 | desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB); |
1000 | } |
1001 | |
1002 | val = rd32(hw, cq->sq.head); |
1003 | if (val >= cq->num_sq_entries) { |
1004 | ice_debug(hw, ICE_DBG_AQ_MSG, "head overrun at %d in the Control Send Queue ring\n" , |
1005 | val); |
1006 | status = -EIO; |
1007 | goto sq_send_command_error; |
1008 | } |
1009 | |
1010 | details = ICE_CTL_Q_DETAILS(cq->sq, cq->sq.next_to_use); |
1011 | if (cd) |
1012 | *details = *cd; |
1013 | else |
1014 | memset(details, 0, sizeof(*details)); |
1015 | |
1016 | /* Call clean and check queue available function to reclaim the |
1017 | * descriptors that were processed by FW/MBX; the function returns the |
1018 | * number of desc available. The clean function called here could be |
1019 | * called in a separate thread in case of asynchronous completions. |
1020 | */ |
1021 | if (ice_clean_sq(hw, cq) == 0) { |
1022 | ice_debug(hw, ICE_DBG_AQ_MSG, "Error: Control Send Queue is full.\n" ); |
1023 | status = -ENOSPC; |
1024 | goto sq_send_command_error; |
1025 | } |
1026 | |
1027 | /* initialize the temp desc pointer with the right desc */ |
1028 | desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use); |
1029 | |
1030 | /* if the desc is available copy the temp desc to the right place */ |
1031 | memcpy(desc_on_ring, desc, sizeof(*desc_on_ring)); |
1032 | |
1033 | /* if buf is not NULL assume indirect command */ |
1034 | if (buf) { |
1035 | dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use]; |
1036 | /* copy the user buf into the respective DMA buf */ |
1037 | memcpy(dma_buf->va, buf, buf_size); |
1038 | desc_on_ring->datalen = cpu_to_le16(buf_size); |
1039 | |
1040 | /* Update the address values in the desc with the pa value |
1041 | * for respective buffer |
1042 | */ |
1043 | desc_on_ring->params.generic.addr_high = |
1044 | cpu_to_le32(upper_32_bits(dma_buf->pa)); |
1045 | desc_on_ring->params.generic.addr_low = |
1046 | cpu_to_le32(lower_32_bits(dma_buf->pa)); |
1047 | } |
1048 | |
1049 | /* Debug desc and buffer */ |
1050 | ice_debug(hw, ICE_DBG_AQ_DESC, "ATQ: Control Send queue desc and buffer:\n" ); |
1051 | |
1052 | ice_debug_cq(hw, desc: (void *)desc_on_ring, buf, buf_len: buf_size); |
1053 | |
1054 | (cq->sq.next_to_use)++; |
1055 | if (cq->sq.next_to_use == cq->sq.count) |
1056 | cq->sq.next_to_use = 0; |
1057 | wr32(hw, cq->sq.tail, cq->sq.next_to_use); |
1058 | ice_flush(hw); |
1059 | |
1060 | /* Wait a short time before initial ice_sq_done() check, to allow |
1061 | * hardware time for completion. |
1062 | */ |
1063 | udelay(5); |
1064 | |
1065 | timeout = jiffies + ICE_CTL_Q_SQ_CMD_TIMEOUT; |
1066 | do { |
1067 | if (ice_sq_done(hw, cq)) |
1068 | break; |
1069 | |
1070 | usleep_range(min: 100, max: 150); |
1071 | } while (time_before(jiffies, timeout)); |
1072 | |
1073 | /* if ready, copy the desc back to temp */ |
1074 | if (ice_sq_done(hw, cq)) { |
1075 | memcpy(desc, desc_on_ring, sizeof(*desc)); |
1076 | if (buf) { |
1077 | /* get returned length to copy */ |
1078 | u16 copy_size = le16_to_cpu(desc->datalen); |
1079 | |
1080 | if (copy_size > buf_size) { |
1081 | ice_debug(hw, ICE_DBG_AQ_MSG, "Return len %d > than buf len %d\n" , |
1082 | copy_size, buf_size); |
1083 | status = -EIO; |
1084 | } else { |
1085 | memcpy(buf, dma_buf->va, copy_size); |
1086 | } |
1087 | } |
1088 | retval = le16_to_cpu(desc->retval); |
1089 | if (retval) { |
1090 | ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue command 0x%04X completed with error 0x%X\n" , |
1091 | le16_to_cpu(desc->opcode), |
1092 | retval); |
1093 | |
1094 | /* strip off FW internal code */ |
1095 | retval &= 0xff; |
1096 | } |
1097 | cmd_completed = true; |
1098 | if (!status && retval != ICE_AQ_RC_OK) |
1099 | status = -EIO; |
1100 | cq->sq_last_status = (enum ice_aq_err)retval; |
1101 | } |
1102 | |
1103 | ice_debug(hw, ICE_DBG_AQ_MSG, "ATQ: desc and buffer writeback:\n" ); |
1104 | |
1105 | ice_debug_cq(hw, desc: (void *)desc, buf, buf_len: buf_size); |
1106 | |
1107 | /* save writeback AQ if requested */ |
1108 | if (details->wb_desc) |
1109 | memcpy(details->wb_desc, desc_on_ring, |
1110 | sizeof(*details->wb_desc)); |
1111 | |
1112 | /* update the error if time out occurred */ |
1113 | if (!cmd_completed) { |
1114 | if (rd32(hw, cq->rq.len) & cq->rq.len_crit_mask || |
1115 | rd32(hw, cq->sq.len) & cq->sq.len_crit_mask) { |
1116 | ice_debug(hw, ICE_DBG_AQ_MSG, "Critical FW error.\n" ); |
1117 | status = -EIO; |
1118 | } else { |
1119 | ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue Writeback timeout.\n" ); |
1120 | status = -EIO; |
1121 | } |
1122 | } |
1123 | |
1124 | sq_send_command_error: |
1125 | mutex_unlock(lock: &cq->sq_lock); |
1126 | return status; |
1127 | } |
1128 | |
1129 | /** |
1130 | * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function |
1131 | * @desc: pointer to the temp descriptor (non DMA mem) |
1132 | * @opcode: the opcode can be used to decide which flags to turn off or on |
1133 | * |
1134 | * Fill the desc with default values |
1135 | */ |
1136 | void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode) |
1137 | { |
1138 | /* zero out the desc */ |
1139 | memset(desc, 0, sizeof(*desc)); |
1140 | desc->opcode = cpu_to_le16(opcode); |
1141 | desc->flags = cpu_to_le16(ICE_AQ_FLAG_SI); |
1142 | } |
1143 | |
1144 | /** |
1145 | * ice_clean_rq_elem |
1146 | * @hw: pointer to the HW struct |
1147 | * @cq: pointer to the specific Control queue |
1148 | * @e: event info from the receive descriptor, includes any buffers |
1149 | * @pending: number of events that could be left to process |
1150 | * |
1151 | * This function cleans one Admin Receive Queue element and returns |
1152 | * the contents through e. It can also return how many events are |
1153 | * left to process through 'pending'. |
1154 | */ |
1155 | int |
1156 | ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq, |
1157 | struct ice_rq_event_info *e, u16 *pending) |
1158 | { |
1159 | u16 ntc = cq->rq.next_to_clean; |
1160 | enum ice_aq_err rq_last_status; |
1161 | struct ice_aq_desc *desc; |
1162 | struct ice_dma_mem *bi; |
1163 | int ret_code = 0; |
1164 | u16 desc_idx; |
1165 | u16 datalen; |
1166 | u16 flags; |
1167 | u16 ntu; |
1168 | |
1169 | /* pre-clean the event info */ |
1170 | memset(&e->desc, 0, sizeof(e->desc)); |
1171 | |
1172 | /* take the lock before we start messing with the ring */ |
1173 | mutex_lock(&cq->rq_lock); |
1174 | |
1175 | if (!cq->rq.count) { |
1176 | ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive queue not initialized.\n" ); |
1177 | ret_code = -EIO; |
1178 | goto clean_rq_elem_err; |
1179 | } |
1180 | |
1181 | /* set next_to_use to head */ |
1182 | ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); |
1183 | |
1184 | if (ntu == ntc) { |
1185 | /* nothing to do - shouldn't need to update ring's values */ |
1186 | ret_code = -EALREADY; |
1187 | goto clean_rq_elem_out; |
1188 | } |
1189 | |
1190 | /* now clean the next descriptor */ |
1191 | desc = ICE_CTL_Q_DESC(cq->rq, ntc); |
1192 | desc_idx = ntc; |
1193 | |
1194 | rq_last_status = (enum ice_aq_err)le16_to_cpu(desc->retval); |
1195 | flags = le16_to_cpu(desc->flags); |
1196 | if (flags & ICE_AQ_FLAG_ERR) { |
1197 | ret_code = -EIO; |
1198 | ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive Queue Event 0x%04X received with error 0x%X\n" , |
1199 | le16_to_cpu(desc->opcode), rq_last_status); |
1200 | } |
1201 | memcpy(&e->desc, desc, sizeof(e->desc)); |
1202 | datalen = le16_to_cpu(desc->datalen); |
1203 | e->msg_len = min_t(u16, datalen, e->buf_len); |
1204 | if (e->msg_buf && e->msg_len) |
1205 | memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va, e->msg_len); |
1206 | |
1207 | ice_debug(hw, ICE_DBG_AQ_DESC, "ARQ: desc and buffer:\n" ); |
1208 | |
1209 | ice_debug_cq(hw, desc: (void *)desc, buf: e->msg_buf, buf_len: cq->rq_buf_size); |
1210 | |
1211 | /* Restore the original datalen and buffer address in the desc, |
1212 | * FW updates datalen to indicate the event message size |
1213 | */ |
1214 | bi = &cq->rq.r.rq_bi[ntc]; |
1215 | memset(desc, 0, sizeof(*desc)); |
1216 | |
1217 | desc->flags = cpu_to_le16(ICE_AQ_FLAG_BUF); |
1218 | if (cq->rq_buf_size > ICE_AQ_LG_BUF) |
1219 | desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB); |
1220 | desc->datalen = cpu_to_le16(bi->size); |
1221 | desc->params.generic.addr_high = cpu_to_le32(upper_32_bits(bi->pa)); |
1222 | desc->params.generic.addr_low = cpu_to_le32(lower_32_bits(bi->pa)); |
1223 | |
1224 | /* set tail = the last cleaned desc index. */ |
1225 | wr32(hw, cq->rq.tail, ntc); |
1226 | /* ntc is updated to tail + 1 */ |
1227 | ntc++; |
1228 | if (ntc == cq->num_rq_entries) |
1229 | ntc = 0; |
1230 | cq->rq.next_to_clean = ntc; |
1231 | cq->rq.next_to_use = ntu; |
1232 | |
1233 | clean_rq_elem_out: |
1234 | /* Set pending if needed, unlock and return */ |
1235 | if (pending) { |
1236 | /* re-read HW head to calculate actual pending messages */ |
1237 | ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); |
1238 | *pending = (u16)((ntc > ntu ? cq->rq.count : 0) + (ntu - ntc)); |
1239 | } |
1240 | clean_rq_elem_err: |
1241 | mutex_unlock(lock: &cq->rq_lock); |
1242 | |
1243 | return ret_code; |
1244 | } |
1245 | |