1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) |
2 | /* QLogic qed NIC Driver |
3 | * Copyright (c) 2015-2017 QLogic Corporation |
4 | * Copyright (c) 2019-2020 Marvell International Ltd. |
5 | */ |
6 | |
7 | #include <linux/types.h> |
8 | #include <linux/io.h> |
9 | #include <linux/delay.h> |
10 | #include <linux/dma-mapping.h> |
11 | #include <linux/errno.h> |
12 | #include <linux/kernel.h> |
13 | #include <linux/list.h> |
14 | #include <linux/mutex.h> |
15 | #include <linux/pci.h> |
16 | #include <linux/slab.h> |
17 | #include <linux/spinlock.h> |
18 | #include <linux/string.h> |
19 | #include <linux/qed/qed_chain.h> |
20 | #include "qed.h" |
21 | #include "qed_hsi.h" |
22 | #include "qed_hw.h" |
23 | #include "qed_reg_addr.h" |
24 | #include "qed_sriov.h" |
25 | |
26 | #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT 1000 |
27 | #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP 1000 |
28 | #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT 100000 |
29 | #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY 10 |
30 | |
31 | /* Invalid values */ |
32 | #define QED_BAR_INVALID_OFFSET (cpu_to_le32(-1)) |
33 | |
34 | struct qed_ptt { |
35 | struct list_head list_entry; |
36 | unsigned int idx; |
37 | struct pxp_ptt_entry pxp; |
38 | u8 hwfn_id; |
39 | }; |
40 | |
41 | struct qed_ptt_pool { |
42 | struct list_head free_list; |
43 | spinlock_t lock; /* ptt synchronized access */ |
44 | struct qed_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM]; |
45 | }; |
46 | |
47 | int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn) |
48 | { |
49 | struct qed_ptt_pool *p_pool = kmalloc(size: sizeof(*p_pool), GFP_KERNEL); |
50 | int i; |
51 | |
52 | if (!p_pool) |
53 | return -ENOMEM; |
54 | |
55 | INIT_LIST_HEAD(list: &p_pool->free_list); |
56 | for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { |
57 | p_pool->ptts[i].idx = i; |
58 | p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET; |
59 | p_pool->ptts[i].pxp.pretend.control = 0; |
60 | p_pool->ptts[i].hwfn_id = p_hwfn->my_id; |
61 | if (i >= RESERVED_PTT_MAX) |
62 | list_add(new: &p_pool->ptts[i].list_entry, |
63 | head: &p_pool->free_list); |
64 | } |
65 | |
66 | p_hwfn->p_ptt_pool = p_pool; |
67 | spin_lock_init(&p_pool->lock); |
68 | |
69 | return 0; |
70 | } |
71 | |
72 | void qed_ptt_invalidate(struct qed_hwfn *p_hwfn) |
73 | { |
74 | struct qed_ptt *p_ptt; |
75 | int i; |
76 | |
77 | for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { |
78 | p_ptt = &p_hwfn->p_ptt_pool->ptts[i]; |
79 | p_ptt->pxp.offset = QED_BAR_INVALID_OFFSET; |
80 | } |
81 | } |
82 | |
83 | void qed_ptt_pool_free(struct qed_hwfn *p_hwfn) |
84 | { |
85 | kfree(objp: p_hwfn->p_ptt_pool); |
86 | p_hwfn->p_ptt_pool = NULL; |
87 | } |
88 | |
89 | struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn) |
90 | { |
91 | return qed_ptt_acquire_context(p_hwfn, is_atomic: false); |
92 | } |
93 | |
94 | struct qed_ptt *qed_ptt_acquire_context(struct qed_hwfn *p_hwfn, bool is_atomic) |
95 | { |
96 | struct qed_ptt *p_ptt; |
97 | unsigned int i, count; |
98 | |
99 | if (is_atomic) |
100 | count = QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT; |
101 | else |
102 | count = QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT; |
103 | |
104 | /* Take the free PTT from the list */ |
105 | for (i = 0; i < count; i++) { |
106 | spin_lock_bh(lock: &p_hwfn->p_ptt_pool->lock); |
107 | |
108 | if (!list_empty(head: &p_hwfn->p_ptt_pool->free_list)) { |
109 | p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list, |
110 | struct qed_ptt, list_entry); |
111 | list_del(entry: &p_ptt->list_entry); |
112 | |
113 | spin_unlock_bh(lock: &p_hwfn->p_ptt_pool->lock); |
114 | |
115 | DP_VERBOSE(p_hwfn, NETIF_MSG_HW, |
116 | "allocated ptt %d\n" , p_ptt->idx); |
117 | return p_ptt; |
118 | } |
119 | |
120 | spin_unlock_bh(lock: &p_hwfn->p_ptt_pool->lock); |
121 | |
122 | if (is_atomic) |
123 | udelay(QED_BAR_ACQUIRE_TIMEOUT_UDELAY); |
124 | else |
125 | usleep_range(QED_BAR_ACQUIRE_TIMEOUT_USLEEP, |
126 | QED_BAR_ACQUIRE_TIMEOUT_USLEEP * 2); |
127 | } |
128 | |
129 | DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n" ); |
130 | return NULL; |
131 | } |
132 | |
133 | void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) |
134 | { |
135 | spin_lock_bh(lock: &p_hwfn->p_ptt_pool->lock); |
136 | list_add(new: &p_ptt->list_entry, head: &p_hwfn->p_ptt_pool->free_list); |
137 | spin_unlock_bh(lock: &p_hwfn->p_ptt_pool->lock); |
138 | } |
139 | |
140 | u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) |
141 | { |
142 | /* The HW is using DWORDS and we need to translate it to Bytes */ |
143 | return le32_to_cpu(p_ptt->pxp.offset) << 2; |
144 | } |
145 | |
146 | static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt) |
147 | { |
148 | return PXP_PF_WINDOW_ADMIN_PER_PF_START + |
149 | p_ptt->idx * sizeof(struct pxp_ptt_entry); |
150 | } |
151 | |
152 | u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt) |
153 | { |
154 | return PXP_EXTERNAL_BAR_PF_WINDOW_START + |
155 | p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE; |
156 | } |
157 | |
158 | void qed_ptt_set_win(struct qed_hwfn *p_hwfn, |
159 | struct qed_ptt *p_ptt, u32 new_hw_addr) |
160 | { |
161 | u32 prev_hw_addr; |
162 | |
163 | prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); |
164 | |
165 | if (new_hw_addr == prev_hw_addr) |
166 | return; |
167 | |
168 | /* Update PTT entery in admin window */ |
169 | DP_VERBOSE(p_hwfn, NETIF_MSG_HW, |
170 | "Updating PTT entry %d to offset 0x%x\n" , |
171 | p_ptt->idx, new_hw_addr); |
172 | |
173 | /* The HW is using DWORDS and the address is in Bytes */ |
174 | p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2); |
175 | |
176 | REG_WR(p_hwfn, |
177 | qed_ptt_config_addr(p_ptt) + |
178 | offsetof(struct pxp_ptt_entry, offset), |
179 | le32_to_cpu(p_ptt->pxp.offset)); |
180 | } |
181 | |
182 | static u32 qed_set_ptt(struct qed_hwfn *p_hwfn, |
183 | struct qed_ptt *p_ptt, u32 hw_addr) |
184 | { |
185 | u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); |
186 | u32 offset; |
187 | |
188 | offset = hw_addr - win_hw_addr; |
189 | |
190 | if (p_ptt->hwfn_id != p_hwfn->my_id) |
191 | DP_NOTICE(p_hwfn, |
192 | "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n" , |
193 | p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id); |
194 | |
195 | /* Verify the address is within the window */ |
196 | if (hw_addr < win_hw_addr || |
197 | offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) { |
198 | qed_ptt_set_win(p_hwfn, p_ptt, new_hw_addr: hw_addr); |
199 | offset = 0; |
200 | } |
201 | |
202 | return qed_ptt_get_bar_addr(p_ptt) + offset; |
203 | } |
204 | |
205 | struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn, |
206 | enum reserved_ptts ptt_idx) |
207 | { |
208 | if (ptt_idx >= RESERVED_PTT_MAX) { |
209 | DP_NOTICE(p_hwfn, |
210 | "Requested PTT %d is out of range\n" , ptt_idx); |
211 | return NULL; |
212 | } |
213 | |
214 | return &p_hwfn->p_ptt_pool->ptts[ptt_idx]; |
215 | } |
216 | |
217 | void qed_wr(struct qed_hwfn *p_hwfn, |
218 | struct qed_ptt *p_ptt, |
219 | u32 hw_addr, u32 val) |
220 | { |
221 | u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); |
222 | |
223 | REG_WR(p_hwfn, bar_addr, val); |
224 | DP_VERBOSE(p_hwfn, NETIF_MSG_HW, |
225 | "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n" , |
226 | bar_addr, hw_addr, val); |
227 | } |
228 | |
229 | u32 qed_rd(struct qed_hwfn *p_hwfn, |
230 | struct qed_ptt *p_ptt, |
231 | u32 hw_addr) |
232 | { |
233 | u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); |
234 | u32 val = REG_RD(p_hwfn, bar_addr); |
235 | |
236 | DP_VERBOSE(p_hwfn, NETIF_MSG_HW, |
237 | "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n" , |
238 | bar_addr, hw_addr, val); |
239 | |
240 | return val; |
241 | } |
242 | |
243 | static void qed_memcpy_hw(struct qed_hwfn *p_hwfn, |
244 | struct qed_ptt *p_ptt, |
245 | void *addr, u32 hw_addr, size_t n, bool to_device) |
246 | { |
247 | u32 dw_count, *host_addr, hw_offset; |
248 | size_t quota, done = 0; |
249 | u32 __iomem *reg_addr; |
250 | |
251 | while (done < n) { |
252 | quota = min_t(size_t, n - done, |
253 | PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE); |
254 | |
255 | if (IS_PF(p_hwfn->cdev)) { |
256 | qed_ptt_set_win(p_hwfn, p_ptt, new_hw_addr: hw_addr + done); |
257 | hw_offset = qed_ptt_get_bar_addr(p_ptt); |
258 | } else { |
259 | hw_offset = hw_addr + done; |
260 | } |
261 | |
262 | dw_count = quota / 4; |
263 | host_addr = (u32 *)((u8 *)addr + done); |
264 | reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset); |
265 | if (to_device) |
266 | while (dw_count--) |
267 | DIRECT_REG_WR(reg_addr++, *host_addr++); |
268 | else |
269 | while (dw_count--) |
270 | *host_addr++ = DIRECT_REG_RD(reg_addr++); |
271 | |
272 | done += quota; |
273 | } |
274 | } |
275 | |
276 | void qed_memcpy_from(struct qed_hwfn *p_hwfn, |
277 | struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n) |
278 | { |
279 | DP_VERBOSE(p_hwfn, NETIF_MSG_HW, |
280 | "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n" , |
281 | hw_addr, dest, hw_addr, (unsigned long)n); |
282 | |
283 | qed_memcpy_hw(p_hwfn, p_ptt, addr: dest, hw_addr, n, to_device: false); |
284 | } |
285 | |
286 | void qed_memcpy_to(struct qed_hwfn *p_hwfn, |
287 | struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n) |
288 | { |
289 | DP_VERBOSE(p_hwfn, NETIF_MSG_HW, |
290 | "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n" , |
291 | hw_addr, hw_addr, src, (unsigned long)n); |
292 | |
293 | qed_memcpy_hw(p_hwfn, p_ptt, addr: src, hw_addr, n, to_device: true); |
294 | } |
295 | |
296 | void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid) |
297 | { |
298 | u16 control = 0; |
299 | |
300 | SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); |
301 | SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); |
302 | |
303 | /* Every pretend undos previous pretends, including |
304 | * previous port pretend. |
305 | */ |
306 | SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); |
307 | SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); |
308 | SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); |
309 | |
310 | if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) |
311 | fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); |
312 | |
313 | p_ptt->pxp.pretend.control = cpu_to_le16(control); |
314 | p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); |
315 | |
316 | REG_WR(p_hwfn, |
317 | qed_ptt_config_addr(p_ptt) + |
318 | offsetof(struct pxp_ptt_entry, pretend), |
319 | *(u32 *)&p_ptt->pxp.pretend); |
320 | } |
321 | |
322 | void qed_port_pretend(struct qed_hwfn *p_hwfn, |
323 | struct qed_ptt *p_ptt, u8 port_id) |
324 | { |
325 | u16 control = 0; |
326 | |
327 | SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); |
328 | SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); |
329 | SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); |
330 | |
331 | p_ptt->pxp.pretend.control = cpu_to_le16(control); |
332 | |
333 | REG_WR(p_hwfn, |
334 | qed_ptt_config_addr(p_ptt) + |
335 | offsetof(struct pxp_ptt_entry, pretend), |
336 | *(u32 *)&p_ptt->pxp.pretend); |
337 | } |
338 | |
339 | void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) |
340 | { |
341 | u16 control = 0; |
342 | |
343 | SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); |
344 | SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); |
345 | SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); |
346 | |
347 | p_ptt->pxp.pretend.control = cpu_to_le16(control); |
348 | |
349 | REG_WR(p_hwfn, |
350 | qed_ptt_config_addr(p_ptt) + |
351 | offsetof(struct pxp_ptt_entry, pretend), |
352 | *(u32 *)&p_ptt->pxp.pretend); |
353 | } |
354 | |
355 | void qed_port_fid_pretend(struct qed_hwfn *p_hwfn, |
356 | struct qed_ptt *p_ptt, u8 port_id, u16 fid) |
357 | { |
358 | u16 control = 0; |
359 | |
360 | SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); |
361 | SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); |
362 | SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); |
363 | SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); |
364 | SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); |
365 | if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) |
366 | fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); |
367 | p_ptt->pxp.pretend.control = cpu_to_le16(control); |
368 | p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); |
369 | REG_WR(p_hwfn, |
370 | qed_ptt_config_addr(p_ptt) + |
371 | offsetof(struct pxp_ptt_entry, pretend), |
372 | *(u32 *)&p_ptt->pxp.pretend); |
373 | } |
374 | |
375 | u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid) |
376 | { |
377 | u32 concrete_fid = 0; |
378 | |
379 | SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id); |
380 | SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid); |
381 | SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1); |
382 | |
383 | return concrete_fid; |
384 | } |
385 | |
386 | /* DMAE */ |
387 | #define QED_DMAE_FLAGS_IS_SET(params, flag) \ |
388 | ((params) != NULL && GET_FIELD((params)->flags, QED_DMAE_PARAMS_##flag)) |
389 | |
390 | static void qed_dmae_opcode(struct qed_hwfn *p_hwfn, |
391 | const u8 is_src_type_grc, |
392 | const u8 is_dst_type_grc, |
393 | struct qed_dmae_params *p_params) |
394 | { |
395 | u8 src_pfid, dst_pfid, port_id; |
396 | u16 opcode_b = 0; |
397 | u32 opcode = 0; |
398 | |
399 | /* Whether the source is the PCIe or the GRC. |
400 | * 0- The source is the PCIe |
401 | * 1- The source is the GRC. |
402 | */ |
403 | SET_FIELD(opcode, DMAE_CMD_SRC, |
404 | (is_src_type_grc ? dmae_cmd_src_grc : dmae_cmd_src_pcie)); |
405 | src_pfid = QED_DMAE_FLAGS_IS_SET(p_params, SRC_PF_VALID) ? |
406 | p_params->src_pfid : p_hwfn->rel_pf_id; |
407 | SET_FIELD(opcode, DMAE_CMD_SRC_PF_ID, src_pfid); |
408 | |
409 | /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */ |
410 | SET_FIELD(opcode, DMAE_CMD_DST, |
411 | (is_dst_type_grc ? dmae_cmd_dst_grc : dmae_cmd_dst_pcie)); |
412 | dst_pfid = QED_DMAE_FLAGS_IS_SET(p_params, DST_PF_VALID) ? |
413 | p_params->dst_pfid : p_hwfn->rel_pf_id; |
414 | SET_FIELD(opcode, DMAE_CMD_DST_PF_ID, dst_pfid); |
415 | |
416 | |
417 | /* Whether to write a completion word to the completion destination: |
418 | * 0-Do not write a completion word |
419 | * 1-Write the completion word |
420 | */ |
421 | SET_FIELD(opcode, DMAE_CMD_COMP_WORD_EN, 1); |
422 | SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1); |
423 | |
424 | if (QED_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST)) |
425 | SET_FIELD(opcode, DMAE_CMD_COMP_FUNC, 1); |
426 | |
427 | /* swapping mode 3 - big endian */ |
428 | SET_FIELD(opcode, DMAE_CMD_ENDIANITY_MODE, DMAE_CMD_ENDIANITY); |
429 | |
430 | port_id = (QED_DMAE_FLAGS_IS_SET(p_params, PORT_VALID)) ? |
431 | p_params->port_id : p_hwfn->port_id; |
432 | SET_FIELD(opcode, DMAE_CMD_PORT_ID, port_id); |
433 | |
434 | /* reset source address in next go */ |
435 | SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1); |
436 | |
437 | /* reset dest address in next go */ |
438 | SET_FIELD(opcode, DMAE_CMD_DST_ADDR_RESET, 1); |
439 | |
440 | /* SRC/DST VFID: all 1's - pf, otherwise VF id */ |
441 | if (QED_DMAE_FLAGS_IS_SET(p_params, SRC_VF_VALID)) { |
442 | SET_FIELD(opcode, DMAE_CMD_SRC_VF_ID_VALID, 1); |
443 | SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, p_params->src_vfid); |
444 | } else { |
445 | SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, 0xFF); |
446 | } |
447 | if (QED_DMAE_FLAGS_IS_SET(p_params, DST_VF_VALID)) { |
448 | SET_FIELD(opcode, DMAE_CMD_DST_VF_ID_VALID, 1); |
449 | SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, p_params->dst_vfid); |
450 | } else { |
451 | SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, 0xFF); |
452 | } |
453 | |
454 | p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode); |
455 | p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b); |
456 | } |
457 | |
458 | u32 qed_dmae_idx_to_go_cmd(u8 idx) |
459 | { |
460 | /* All the DMAE 'go' registers form an array in internal memory */ |
461 | return DMAE_REG_GO_C0 + (idx << 2); |
462 | } |
463 | |
464 | static int qed_dmae_post_command(struct qed_hwfn *p_hwfn, |
465 | struct qed_ptt *p_ptt) |
466 | { |
467 | struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd; |
468 | u8 idx_cmd = p_hwfn->dmae_info.channel, i; |
469 | int qed_status = 0; |
470 | |
471 | /* verify address is not NULL */ |
472 | if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) || |
473 | ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) { |
474 | DP_NOTICE(p_hwfn, |
475 | "source or destination address 0 idx_cmd=%d\n" |
476 | "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n" , |
477 | idx_cmd, |
478 | le32_to_cpu(p_command->opcode), |
479 | le16_to_cpu(p_command->opcode_b), |
480 | le16_to_cpu(p_command->length_dw), |
481 | le32_to_cpu(p_command->src_addr_hi), |
482 | le32_to_cpu(p_command->src_addr_lo), |
483 | le32_to_cpu(p_command->dst_addr_hi), |
484 | le32_to_cpu(p_command->dst_addr_lo)); |
485 | |
486 | return -EINVAL; |
487 | } |
488 | |
489 | DP_VERBOSE(p_hwfn, |
490 | NETIF_MSG_HW, |
491 | "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n" , |
492 | idx_cmd, |
493 | le32_to_cpu(p_command->opcode), |
494 | le16_to_cpu(p_command->opcode_b), |
495 | le16_to_cpu(p_command->length_dw), |
496 | le32_to_cpu(p_command->src_addr_hi), |
497 | le32_to_cpu(p_command->src_addr_lo), |
498 | le32_to_cpu(p_command->dst_addr_hi), |
499 | le32_to_cpu(p_command->dst_addr_lo)); |
500 | |
501 | /* Copy the command to DMAE - need to do it before every call |
502 | * for source/dest address no reset. |
503 | * The first 9 DWs are the command registers, the 10 DW is the |
504 | * GO register, and the rest are result registers |
505 | * (which are read only by the client). |
506 | */ |
507 | for (i = 0; i < DMAE_CMD_SIZE; i++) { |
508 | u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ? |
509 | *(((u32 *)p_command) + i) : 0; |
510 | |
511 | qed_wr(p_hwfn, p_ptt, |
512 | DMAE_REG_CMD_MEM + |
513 | (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) + |
514 | (i * sizeof(u32)), val: data); |
515 | } |
516 | |
517 | qed_wr(p_hwfn, p_ptt, hw_addr: qed_dmae_idx_to_go_cmd(idx: idx_cmd), DMAE_GO_VALUE); |
518 | |
519 | return qed_status; |
520 | } |
521 | |
522 | int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn) |
523 | { |
524 | dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr; |
525 | struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd; |
526 | u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer; |
527 | u32 **p_comp = &p_hwfn->dmae_info.p_completion_word; |
528 | |
529 | *p_comp = dma_alloc_coherent(dev: &p_hwfn->cdev->pdev->dev, |
530 | size: sizeof(u32), dma_handle: p_addr, GFP_KERNEL); |
531 | if (!*p_comp) |
532 | goto err; |
533 | |
534 | p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr; |
535 | *p_cmd = dma_alloc_coherent(dev: &p_hwfn->cdev->pdev->dev, |
536 | size: sizeof(struct dmae_cmd), |
537 | dma_handle: p_addr, GFP_KERNEL); |
538 | if (!*p_cmd) |
539 | goto err; |
540 | |
541 | p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr; |
542 | *p_buff = dma_alloc_coherent(dev: &p_hwfn->cdev->pdev->dev, |
543 | size: sizeof(u32) * DMAE_MAX_RW_SIZE, |
544 | dma_handle: p_addr, GFP_KERNEL); |
545 | if (!*p_buff) |
546 | goto err; |
547 | |
548 | p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id; |
549 | |
550 | return 0; |
551 | err: |
552 | qed_dmae_info_free(p_hwfn); |
553 | return -ENOMEM; |
554 | } |
555 | |
556 | void qed_dmae_info_free(struct qed_hwfn *p_hwfn) |
557 | { |
558 | dma_addr_t p_phys; |
559 | |
560 | /* Just make sure no one is in the middle */ |
561 | mutex_lock(&p_hwfn->dmae_info.mutex); |
562 | |
563 | if (p_hwfn->dmae_info.p_completion_word) { |
564 | p_phys = p_hwfn->dmae_info.completion_word_phys_addr; |
565 | dma_free_coherent(dev: &p_hwfn->cdev->pdev->dev, |
566 | size: sizeof(u32), |
567 | cpu_addr: p_hwfn->dmae_info.p_completion_word, dma_handle: p_phys); |
568 | p_hwfn->dmae_info.p_completion_word = NULL; |
569 | } |
570 | |
571 | if (p_hwfn->dmae_info.p_dmae_cmd) { |
572 | p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr; |
573 | dma_free_coherent(dev: &p_hwfn->cdev->pdev->dev, |
574 | size: sizeof(struct dmae_cmd), |
575 | cpu_addr: p_hwfn->dmae_info.p_dmae_cmd, dma_handle: p_phys); |
576 | p_hwfn->dmae_info.p_dmae_cmd = NULL; |
577 | } |
578 | |
579 | if (p_hwfn->dmae_info.p_intermediate_buffer) { |
580 | p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; |
581 | dma_free_coherent(dev: &p_hwfn->cdev->pdev->dev, |
582 | size: sizeof(u32) * DMAE_MAX_RW_SIZE, |
583 | cpu_addr: p_hwfn->dmae_info.p_intermediate_buffer, |
584 | dma_handle: p_phys); |
585 | p_hwfn->dmae_info.p_intermediate_buffer = NULL; |
586 | } |
587 | |
588 | mutex_unlock(lock: &p_hwfn->dmae_info.mutex); |
589 | } |
590 | |
591 | static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn) |
592 | { |
593 | u32 wait_cnt_limit = 10000, wait_cnt = 0; |
594 | int qed_status = 0; |
595 | |
596 | barrier(); |
597 | while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) { |
598 | udelay(DMAE_MIN_WAIT_TIME); |
599 | if (++wait_cnt > wait_cnt_limit) { |
600 | DP_NOTICE(p_hwfn->cdev, |
601 | "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n" , |
602 | *p_hwfn->dmae_info.p_completion_word, |
603 | DMAE_COMPLETION_VAL); |
604 | qed_status = -EBUSY; |
605 | break; |
606 | } |
607 | |
608 | /* to sync the completion_word since we are not |
609 | * using the volatile keyword for p_completion_word |
610 | */ |
611 | barrier(); |
612 | } |
613 | |
614 | if (qed_status == 0) |
615 | *p_hwfn->dmae_info.p_completion_word = 0; |
616 | |
617 | return qed_status; |
618 | } |
619 | |
620 | static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn, |
621 | struct qed_ptt *p_ptt, |
622 | u64 src_addr, |
623 | u64 dst_addr, |
624 | u8 src_type, |
625 | u8 dst_type, |
626 | u32 length_dw) |
627 | { |
628 | dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; |
629 | struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; |
630 | int qed_status = 0; |
631 | |
632 | switch (src_type) { |
633 | case QED_DMAE_ADDRESS_GRC: |
634 | case QED_DMAE_ADDRESS_HOST_PHYS: |
635 | cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr)); |
636 | cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr)); |
637 | break; |
638 | /* for virtual source addresses we use the intermediate buffer. */ |
639 | case QED_DMAE_ADDRESS_HOST_VIRT: |
640 | cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys)); |
641 | cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys)); |
642 | memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0], |
643 | (void *)(uintptr_t)src_addr, |
644 | length_dw * sizeof(u32)); |
645 | break; |
646 | default: |
647 | return -EINVAL; |
648 | } |
649 | |
650 | switch (dst_type) { |
651 | case QED_DMAE_ADDRESS_GRC: |
652 | case QED_DMAE_ADDRESS_HOST_PHYS: |
653 | cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr)); |
654 | cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr)); |
655 | break; |
656 | /* for virtual source addresses we use the intermediate buffer. */ |
657 | case QED_DMAE_ADDRESS_HOST_VIRT: |
658 | cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys)); |
659 | cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys)); |
660 | break; |
661 | default: |
662 | return -EINVAL; |
663 | } |
664 | |
665 | cmd->length_dw = cpu_to_le16((u16)length_dw); |
666 | |
667 | qed_dmae_post_command(p_hwfn, p_ptt); |
668 | |
669 | qed_status = qed_dmae_operation_wait(p_hwfn); |
670 | |
671 | if (qed_status) { |
672 | DP_NOTICE(p_hwfn, |
673 | "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n" , |
674 | src_addr, dst_addr, length_dw); |
675 | return qed_status; |
676 | } |
677 | |
678 | if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT) |
679 | memcpy((void *)(uintptr_t)(dst_addr), |
680 | &p_hwfn->dmae_info.p_intermediate_buffer[0], |
681 | length_dw * sizeof(u32)); |
682 | |
683 | return 0; |
684 | } |
685 | |
686 | static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn, |
687 | struct qed_ptt *p_ptt, |
688 | u64 src_addr, u64 dst_addr, |
689 | u8 src_type, u8 dst_type, |
690 | u32 size_in_dwords, |
691 | struct qed_dmae_params *p_params) |
692 | { |
693 | dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr; |
694 | u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0; |
695 | struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; |
696 | u64 src_addr_split = 0, dst_addr_split = 0; |
697 | u16 length_limit = DMAE_MAX_RW_SIZE; |
698 | int qed_status = 0; |
699 | u32 offset = 0; |
700 | |
701 | if (p_hwfn->cdev->recov_in_prog) { |
702 | DP_VERBOSE(p_hwfn, |
703 | NETIF_MSG_HW, |
704 | "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n" , |
705 | src_addr, src_type, dst_addr, dst_type, |
706 | size_in_dwords); |
707 | |
708 | /* Let the flow complete w/o any error handling */ |
709 | return 0; |
710 | } |
711 | |
712 | qed_dmae_opcode(p_hwfn, |
713 | is_src_type_grc: (src_type == QED_DMAE_ADDRESS_GRC), |
714 | is_dst_type_grc: (dst_type == QED_DMAE_ADDRESS_GRC), |
715 | p_params); |
716 | |
717 | cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys)); |
718 | cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys)); |
719 | cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL); |
720 | |
721 | /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */ |
722 | cnt_split = size_in_dwords / length_limit; |
723 | length_mod = size_in_dwords % length_limit; |
724 | |
725 | src_addr_split = src_addr; |
726 | dst_addr_split = dst_addr; |
727 | |
728 | for (i = 0; i <= cnt_split; i++) { |
729 | offset = length_limit * i; |
730 | |
731 | if (!QED_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) { |
732 | if (src_type == QED_DMAE_ADDRESS_GRC) |
733 | src_addr_split = src_addr + offset; |
734 | else |
735 | src_addr_split = src_addr + (offset * 4); |
736 | } |
737 | |
738 | if (dst_type == QED_DMAE_ADDRESS_GRC) |
739 | dst_addr_split = dst_addr + offset; |
740 | else |
741 | dst_addr_split = dst_addr + (offset * 4); |
742 | |
743 | length_cur = (cnt_split == i) ? length_mod : length_limit; |
744 | |
745 | /* might be zero on last iteration */ |
746 | if (!length_cur) |
747 | continue; |
748 | |
749 | qed_status = qed_dmae_execute_sub_operation(p_hwfn, |
750 | p_ptt, |
751 | src_addr: src_addr_split, |
752 | dst_addr: dst_addr_split, |
753 | src_type, |
754 | dst_type, |
755 | length_dw: length_cur); |
756 | if (qed_status) { |
757 | qed_hw_err_notify(p_hwfn, p_ptt, err_type: QED_HW_ERR_DMAE_FAIL, |
758 | fmt: "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n" , |
759 | qed_status, src_addr, |
760 | dst_addr, length_cur); |
761 | break; |
762 | } |
763 | } |
764 | |
765 | return qed_status; |
766 | } |
767 | |
768 | int qed_dmae_host2grc(struct qed_hwfn *p_hwfn, |
769 | struct qed_ptt *p_ptt, |
770 | u64 source_addr, u32 grc_addr, u32 size_in_dwords, |
771 | struct qed_dmae_params *p_params) |
772 | { |
773 | u32 grc_addr_in_dw = grc_addr / sizeof(u32); |
774 | int rc; |
775 | |
776 | |
777 | mutex_lock(&p_hwfn->dmae_info.mutex); |
778 | |
779 | rc = qed_dmae_execute_command(p_hwfn, p_ptt, src_addr: source_addr, |
780 | dst_addr: grc_addr_in_dw, |
781 | src_type: QED_DMAE_ADDRESS_HOST_VIRT, |
782 | dst_type: QED_DMAE_ADDRESS_GRC, |
783 | size_in_dwords, p_params); |
784 | |
785 | mutex_unlock(lock: &p_hwfn->dmae_info.mutex); |
786 | |
787 | return rc; |
788 | } |
789 | |
790 | int qed_dmae_grc2host(struct qed_hwfn *p_hwfn, |
791 | struct qed_ptt *p_ptt, |
792 | u32 grc_addr, |
793 | dma_addr_t dest_addr, u32 size_in_dwords, |
794 | struct qed_dmae_params *p_params) |
795 | { |
796 | u32 grc_addr_in_dw = grc_addr / sizeof(u32); |
797 | int rc; |
798 | |
799 | |
800 | mutex_lock(&p_hwfn->dmae_info.mutex); |
801 | |
802 | rc = qed_dmae_execute_command(p_hwfn, p_ptt, src_addr: grc_addr_in_dw, |
803 | dst_addr: dest_addr, src_type: QED_DMAE_ADDRESS_GRC, |
804 | dst_type: QED_DMAE_ADDRESS_HOST_VIRT, |
805 | size_in_dwords, p_params); |
806 | |
807 | mutex_unlock(lock: &p_hwfn->dmae_info.mutex); |
808 | |
809 | return rc; |
810 | } |
811 | |
812 | int qed_dmae_host2host(struct qed_hwfn *p_hwfn, |
813 | struct qed_ptt *p_ptt, |
814 | dma_addr_t source_addr, |
815 | dma_addr_t dest_addr, |
816 | u32 size_in_dwords, struct qed_dmae_params *p_params) |
817 | { |
818 | int rc; |
819 | |
820 | mutex_lock(&(p_hwfn->dmae_info.mutex)); |
821 | |
822 | rc = qed_dmae_execute_command(p_hwfn, p_ptt, src_addr: source_addr, |
823 | dst_addr: dest_addr, |
824 | src_type: QED_DMAE_ADDRESS_HOST_PHYS, |
825 | dst_type: QED_DMAE_ADDRESS_HOST_PHYS, |
826 | size_in_dwords, p_params); |
827 | |
828 | mutex_unlock(lock: &(p_hwfn->dmae_info.mutex)); |
829 | |
830 | return rc; |
831 | } |
832 | |
833 | void qed_hw_err_notify(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, |
834 | enum qed_hw_err_type err_type, const char *fmt, ...) |
835 | { |
836 | char buf[QED_HW_ERR_MAX_STR_SIZE]; |
837 | va_list vl; |
838 | int len; |
839 | |
840 | if (fmt) { |
841 | va_start(vl, fmt); |
842 | len = vsnprintf(buf, QED_HW_ERR_MAX_STR_SIZE, fmt, args: vl); |
843 | va_end(vl); |
844 | |
845 | if (len > QED_HW_ERR_MAX_STR_SIZE - 1) |
846 | len = QED_HW_ERR_MAX_STR_SIZE - 1; |
847 | |
848 | DP_NOTICE(p_hwfn, "%s" , buf); |
849 | } |
850 | |
851 | /* Fan failure cannot be masked by handling of another HW error */ |
852 | if (p_hwfn->cdev->recov_in_prog && |
853 | err_type != QED_HW_ERR_FAN_FAIL) { |
854 | DP_VERBOSE(p_hwfn, |
855 | NETIF_MSG_DRV, |
856 | "Recovery is in progress. Avoid notifying about HW error %d.\n" , |
857 | err_type); |
858 | return; |
859 | } |
860 | |
861 | qed_hw_error_occurred(p_hwfn, err_type); |
862 | |
863 | if (fmt) |
864 | qed_mcp_send_raw_debug_data(p_hwfn, p_ptt, p_buf: buf, size: len); |
865 | } |
866 | |
867 | int qed_dmae_sanity(struct qed_hwfn *p_hwfn, |
868 | struct qed_ptt *p_ptt, const char *phase) |
869 | { |
870 | u32 size = PAGE_SIZE / 2, val; |
871 | int rc = 0; |
872 | dma_addr_t p_phys; |
873 | void *p_virt; |
874 | u32 *p_tmp; |
875 | |
876 | p_virt = dma_alloc_coherent(dev: &p_hwfn->cdev->pdev->dev, |
877 | size: 2 * size, dma_handle: &p_phys, GFP_KERNEL); |
878 | if (!p_virt) { |
879 | DP_NOTICE(p_hwfn, |
880 | "DMAE sanity [%s]: failed to allocate memory\n" , |
881 | phase); |
882 | return -ENOMEM; |
883 | } |
884 | |
885 | /* Fill the bottom half of the allocated memory with a known pattern */ |
886 | for (p_tmp = (u32 *)p_virt; |
887 | p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) { |
888 | /* Save the address itself as the value */ |
889 | val = (u32)(uintptr_t)p_tmp; |
890 | *p_tmp = val; |
891 | } |
892 | |
893 | /* Zero the top half of the allocated memory */ |
894 | memset((u8 *)p_virt + size, 0, size); |
895 | |
896 | DP_VERBOSE(p_hwfn, |
897 | QED_MSG_SP, |
898 | "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n" , |
899 | phase, |
900 | (u64)p_phys, |
901 | p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size); |
902 | |
903 | rc = qed_dmae_host2host(p_hwfn, p_ptt, source_addr: p_phys, dest_addr: p_phys + size, |
904 | size_in_dwords: size / 4, NULL); |
905 | if (rc) { |
906 | DP_NOTICE(p_hwfn, |
907 | "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n" , |
908 | phase, rc); |
909 | goto out; |
910 | } |
911 | |
912 | /* Verify that the top half of the allocated memory has the pattern */ |
913 | for (p_tmp = (u32 *)((u8 *)p_virt + size); |
914 | p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) { |
915 | /* The corresponding address in the bottom half */ |
916 | val = (u32)(uintptr_t)p_tmp - size; |
917 | |
918 | if (*p_tmp != val) { |
919 | DP_NOTICE(p_hwfn, |
920 | "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n" , |
921 | phase, |
922 | (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt), |
923 | p_tmp, *p_tmp, val); |
924 | rc = -EINVAL; |
925 | goto out; |
926 | } |
927 | } |
928 | |
929 | out: |
930 | dma_free_coherent(dev: &p_hwfn->cdev->pdev->dev, size: 2 * size, cpu_addr: p_virt, dma_handle: p_phys); |
931 | return rc; |
932 | } |
933 | |