1 | /* |
2 | * Copyright 2021 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | |
24 | #include <linux/pci.h> |
25 | |
26 | #include "amdgpu.h" |
27 | #include "amdgpu_ih.h" |
28 | |
29 | #include "oss/osssys_6_0_0_offset.h" |
30 | #include "oss/osssys_6_0_0_sh_mask.h" |
31 | |
32 | #include "soc15_common.h" |
33 | #include "ih_v6_0.h" |
34 | |
35 | #define MAX_REARM_RETRY 10 |
36 | |
37 | static void ih_v6_0_set_interrupt_funcs(struct amdgpu_device *adev); |
38 | |
39 | /** |
40 | * ih_v6_0_init_register_offset - Initialize register offset for ih rings |
41 | * |
42 | * @adev: amdgpu_device pointer |
43 | * |
44 | * Initialize register offset ih rings (IH_V6_0). |
45 | */ |
46 | static void ih_v6_0_init_register_offset(struct amdgpu_device *adev) |
47 | { |
48 | struct amdgpu_ih_regs *ih_regs; |
49 | |
50 | /* ih ring 2 is removed |
51 | * ih ring and ih ring 1 are available */ |
52 | if (adev->irq.ih.ring_size) { |
53 | ih_regs = &adev->irq.ih.ih_regs; |
54 | ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_BASE); |
55 | ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_BASE_HI); |
56 | ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_CNTL); |
57 | ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_WPTR); |
58 | ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_RPTR); |
59 | ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_DOORBELL_RPTR); |
60 | ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_WPTR_ADDR_LO); |
61 | ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_WPTR_ADDR_HI); |
62 | ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL; |
63 | } |
64 | |
65 | if (adev->irq.ih1.ring_size) { |
66 | ih_regs = &adev->irq.ih1.ih_regs; |
67 | ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_BASE_RING1); |
68 | ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_BASE_HI_RING1); |
69 | ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_CNTL_RING1); |
70 | ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_WPTR_RING1); |
71 | ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_RB_RPTR_RING1); |
72 | ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, regIH_DOORBELL_RPTR_RING1); |
73 | ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1; |
74 | } |
75 | } |
76 | |
77 | /** |
78 | * force_update_wptr_for_self_int - Force update the wptr for self interrupt |
79 | * |
80 | * @adev: amdgpu_device pointer |
81 | * @threshold: threshold to trigger the wptr reporting |
82 | * @timeout: timeout to trigger the wptr reporting |
83 | * @enabled: Enable/disable timeout flush mechanism |
84 | * |
85 | * threshold input range: 0 ~ 15, default 0, |
86 | * real_threshold = 2^threshold |
87 | * timeout input range: 0 ~ 20, default 8, |
88 | * real_timeout = (2^timeout) * 1024 / (socclk_freq) |
89 | * |
90 | * Force update wptr for self interrupt ( >= SIENNA_CICHLID). |
91 | */ |
92 | static void |
93 | force_update_wptr_for_self_int(struct amdgpu_device *adev, |
94 | u32 threshold, u32 timeout, bool enabled) |
95 | { |
96 | u32 ih_cntl, ih_rb_cntl; |
97 | |
98 | ih_cntl = RREG32_SOC15(OSSSYS, 0, regIH_CNTL2); |
99 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, regIH_RB_CNTL_RING1); |
100 | |
101 | ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL2, |
102 | SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT, timeout); |
103 | ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL2, |
104 | SELF_IV_FORCE_WPTR_UPDATE_ENABLE, enabled); |
105 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, |
106 | RB_USED_INT_THRESHOLD, threshold); |
107 | |
108 | if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) { |
109 | if (psp_reg_program(psp: &adev->psp, reg: PSP_REG_IH_RB_CNTL_RING1, value: ih_rb_cntl)) |
110 | return; |
111 | } else { |
112 | WREG32_SOC15(OSSSYS, 0, regIH_RB_CNTL_RING1, ih_rb_cntl); |
113 | } |
114 | |
115 | WREG32_SOC15(OSSSYS, 0, regIH_CNTL2, ih_cntl); |
116 | } |
117 | |
118 | /** |
119 | * ih_v6_0_toggle_ring_interrupts - toggle the interrupt ring buffer |
120 | * |
121 | * @adev: amdgpu_device pointer |
122 | * @ih: amdgpu_ih_ring pointer |
123 | * @enable: true - enable the interrupts, false - disable the interrupts |
124 | * |
125 | * Toggle the interrupt ring buffer (IH_V6_0) |
126 | */ |
127 | static int ih_v6_0_toggle_ring_interrupts(struct amdgpu_device *adev, |
128 | struct amdgpu_ih_ring *ih, |
129 | bool enable) |
130 | { |
131 | struct amdgpu_ih_regs *ih_regs; |
132 | uint32_t tmp; |
133 | |
134 | ih_regs = &ih->ih_regs; |
135 | |
136 | tmp = RREG32(ih_regs->ih_rb_cntl); |
137 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); |
138 | /* enable_intr field is only valid in ring0 */ |
139 | if (ih == &adev->irq.ih) |
140 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); |
141 | |
142 | if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) { |
143 | if (psp_reg_program(psp: &adev->psp, reg: ih_regs->psp_reg_id, value: tmp)) |
144 | return -ETIMEDOUT; |
145 | } else { |
146 | WREG32(ih_regs->ih_rb_cntl, tmp); |
147 | } |
148 | |
149 | if (enable) { |
150 | ih->enabled = true; |
151 | } else { |
152 | /* set rptr, wptr to 0 */ |
153 | WREG32(ih_regs->ih_rb_rptr, 0); |
154 | WREG32(ih_regs->ih_rb_wptr, 0); |
155 | ih->enabled = false; |
156 | ih->rptr = 0; |
157 | } |
158 | |
159 | return 0; |
160 | } |
161 | |
162 | /** |
163 | * ih_v6_0_toggle_interrupts - Toggle all the available interrupt ring buffers |
164 | * |
165 | * @adev: amdgpu_device pointer |
166 | * @enable: enable or disable interrupt ring buffers |
167 | * |
168 | * Toggle all the available interrupt ring buffers (IH_V6_0). |
169 | */ |
170 | static int ih_v6_0_toggle_interrupts(struct amdgpu_device *adev, bool enable) |
171 | { |
172 | struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1}; |
173 | int i; |
174 | int r; |
175 | |
176 | for (i = 0; i < ARRAY_SIZE(ih); i++) { |
177 | if (ih[i]->ring_size) { |
178 | r = ih_v6_0_toggle_ring_interrupts(adev, ih: ih[i], enable); |
179 | if (r) |
180 | return r; |
181 | } |
182 | } |
183 | |
184 | return 0; |
185 | } |
186 | |
187 | static uint32_t ih_v6_0_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) |
188 | { |
189 | int rb_bufsz = order_base_2(ih->ring_size / 4); |
190 | |
191 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, |
192 | MC_SPACE, ih->use_bus_addr ? 2 : 4); |
193 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, |
194 | WPTR_OVERFLOW_CLEAR, 1); |
195 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, |
196 | WPTR_OVERFLOW_ENABLE, 1); |
197 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); |
198 | /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register |
199 | * value is written to memory |
200 | */ |
201 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, |
202 | WPTR_WRITEBACK_ENABLE, 1); |
203 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1); |
204 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0); |
205 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0); |
206 | |
207 | return ih_rb_cntl; |
208 | } |
209 | |
210 | static uint32_t ih_v6_0_doorbell_rptr(struct amdgpu_ih_ring *ih) |
211 | { |
212 | u32 ih_doorbell_rtpr = 0; |
213 | |
214 | if (ih->use_doorbell) { |
215 | ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, |
216 | IH_DOORBELL_RPTR, OFFSET, |
217 | ih->doorbell_index); |
218 | ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, |
219 | IH_DOORBELL_RPTR, |
220 | ENABLE, 1); |
221 | } else { |
222 | ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, |
223 | IH_DOORBELL_RPTR, |
224 | ENABLE, 0); |
225 | } |
226 | return ih_doorbell_rtpr; |
227 | } |
228 | |
229 | /** |
230 | * ih_v6_0_enable_ring - enable an ih ring buffer |
231 | * |
232 | * @adev: amdgpu_device pointer |
233 | * @ih: amdgpu_ih_ring pointer |
234 | * |
235 | * Enable an ih ring buffer (IH_V6_0) |
236 | */ |
237 | static int ih_v6_0_enable_ring(struct amdgpu_device *adev, |
238 | struct amdgpu_ih_ring *ih) |
239 | { |
240 | struct amdgpu_ih_regs *ih_regs; |
241 | uint32_t tmp; |
242 | |
243 | ih_regs = &ih->ih_regs; |
244 | |
245 | /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ |
246 | WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8); |
247 | WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff); |
248 | |
249 | tmp = RREG32(ih_regs->ih_rb_cntl); |
250 | tmp = ih_v6_0_rb_cntl(ih, ih_rb_cntl: tmp); |
251 | if (ih == &adev->irq.ih) |
252 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); |
253 | if (ih == &adev->irq.ih1) { |
254 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); |
255 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); |
256 | } |
257 | |
258 | if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) { |
259 | if (psp_reg_program(psp: &adev->psp, reg: ih_regs->psp_reg_id, value: tmp)) { |
260 | DRM_ERROR("PSP program IH_RB_CNTL failed!\n" ); |
261 | return -ETIMEDOUT; |
262 | } |
263 | } else { |
264 | WREG32(ih_regs->ih_rb_cntl, tmp); |
265 | } |
266 | |
267 | if (ih == &adev->irq.ih) { |
268 | /* set the ih ring 0 writeback address whether it's enabled or not */ |
269 | WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr)); |
270 | WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF); |
271 | } |
272 | |
273 | /* set rptr, wptr to 0 */ |
274 | WREG32(ih_regs->ih_rb_wptr, 0); |
275 | WREG32(ih_regs->ih_rb_rptr, 0); |
276 | |
277 | WREG32(ih_regs->ih_doorbell_rptr, ih_v6_0_doorbell_rptr(ih)); |
278 | |
279 | return 0; |
280 | } |
281 | |
282 | /** |
283 | * ih_v6_0_irq_init - init and enable the interrupt ring |
284 | * |
285 | * @adev: amdgpu_device pointer |
286 | * |
287 | * Allocate a ring buffer for the interrupt controller, |
288 | * enable the RLC, disable interrupts, enable the IH |
289 | * ring buffer and enable it. |
290 | * Called at device load and reume. |
291 | * Returns 0 for success, errors for failure. |
292 | */ |
293 | static int ih_v6_0_irq_init(struct amdgpu_device *adev) |
294 | { |
295 | struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1}; |
296 | u32 ih_chicken; |
297 | u32 tmp; |
298 | int ret; |
299 | int i; |
300 | |
301 | /* disable irqs */ |
302 | ret = ih_v6_0_toggle_interrupts(adev, enable: false); |
303 | if (ret) |
304 | return ret; |
305 | |
306 | adev->nbio.funcs->ih_control(adev); |
307 | |
308 | if (unlikely((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) || |
309 | (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO))) { |
310 | if (ih[0]->use_bus_addr) { |
311 | ih_chicken = RREG32_SOC15(OSSSYS, 0, regIH_CHICKEN); |
312 | ih_chicken = REG_SET_FIELD(ih_chicken, |
313 | IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1); |
314 | WREG32_SOC15(OSSSYS, 0, regIH_CHICKEN, ih_chicken); |
315 | } |
316 | } |
317 | |
318 | for (i = 0; i < ARRAY_SIZE(ih); i++) { |
319 | if (ih[i]->ring_size) { |
320 | ret = ih_v6_0_enable_ring(adev, ih: ih[i]); |
321 | if (ret) |
322 | return ret; |
323 | } |
324 | } |
325 | |
326 | /* update doorbell range for ih ring 0 */ |
327 | adev->nbio.funcs->ih_doorbell_range(adev, ih[0]->use_doorbell, |
328 | ih[0]->doorbell_index); |
329 | |
330 | tmp = RREG32_SOC15(OSSSYS, 0, regIH_STORM_CLIENT_LIST_CNTL); |
331 | tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, |
332 | CLIENT18_IS_STORM_CLIENT, 1); |
333 | WREG32_SOC15(OSSSYS, 0, regIH_STORM_CLIENT_LIST_CNTL, tmp); |
334 | |
335 | tmp = RREG32_SOC15(OSSSYS, 0, regIH_INT_FLOOD_CNTL); |
336 | tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1); |
337 | WREG32_SOC15(OSSSYS, 0, regIH_INT_FLOOD_CNTL, tmp); |
338 | |
339 | /* GC/MMHUB UTCL2 page fault interrupts are configured as |
340 | * MSI storm capable interrupts by deafult. The delay is |
341 | * used to avoid ISR being called too frequently |
342 | * when page fault happens on several continuous page |
343 | * and thus avoid MSI storm */ |
344 | tmp = RREG32_SOC15(OSSSYS, 0, regIH_MSI_STORM_CTRL); |
345 | tmp = REG_SET_FIELD(tmp, IH_MSI_STORM_CTRL, |
346 | DELAY, 3); |
347 | WREG32_SOC15(OSSSYS, 0, regIH_MSI_STORM_CTRL, tmp); |
348 | |
349 | pci_set_master(dev: adev->pdev); |
350 | |
351 | /* enable interrupts */ |
352 | ret = ih_v6_0_toggle_interrupts(adev, enable: true); |
353 | if (ret) |
354 | return ret; |
355 | /* enable wptr force update for self int */ |
356 | force_update_wptr_for_self_int(adev, threshold: 0, timeout: 8, enabled: true); |
357 | |
358 | if (adev->irq.ih_soft.ring_size) |
359 | adev->irq.ih_soft.enabled = true; |
360 | |
361 | return 0; |
362 | } |
363 | |
364 | /** |
365 | * ih_v6_0_irq_disable - disable interrupts |
366 | * |
367 | * @adev: amdgpu_device pointer |
368 | * |
369 | * Disable interrupts on the hw. |
370 | */ |
371 | static void ih_v6_0_irq_disable(struct amdgpu_device *adev) |
372 | { |
373 | force_update_wptr_for_self_int(adev, threshold: 0, timeout: 8, enabled: false); |
374 | ih_v6_0_toggle_interrupts(adev, enable: false); |
375 | |
376 | /* Wait and acknowledge irq */ |
377 | mdelay(1); |
378 | } |
379 | |
380 | /** |
381 | * ih_v6_0_get_wptr - get the IH ring buffer wptr |
382 | * |
383 | * @adev: amdgpu_device pointer |
384 | * @ih: amdgpu_ih_ring pointer |
385 | * |
386 | * Get the IH ring buffer wptr from either the register |
387 | * or the writeback memory buffer. Also check for |
388 | * ring buffer overflow and deal with it. |
389 | * Returns the value of the wptr. |
390 | */ |
391 | static u32 ih_v6_0_get_wptr(struct amdgpu_device *adev, |
392 | struct amdgpu_ih_ring *ih) |
393 | { |
394 | u32 wptr, tmp; |
395 | struct amdgpu_ih_regs *ih_regs; |
396 | |
397 | wptr = le32_to_cpu(*ih->wptr_cpu); |
398 | ih_regs = &ih->ih_regs; |
399 | |
400 | if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) |
401 | goto out; |
402 | |
403 | wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); |
404 | if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) |
405 | goto out; |
406 | wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); |
407 | |
408 | /* When a ring buffer overflow happen start parsing interrupt |
409 | * from the last not overwritten vector (wptr + 32). Hopefully |
410 | * this should allow us to catch up. |
411 | */ |
412 | tmp = (wptr + 32) & ih->ptr_mask; |
413 | dev_warn(adev->dev, "IH ring buffer overflow " |
414 | "(0x%08X, 0x%08X, 0x%08X)\n" , |
415 | wptr, ih->rptr, tmp); |
416 | ih->rptr = tmp; |
417 | |
418 | tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl); |
419 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); |
420 | WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); |
421 | |
422 | /* Unset the CLEAR_OVERFLOW bit immediately so new overflows |
423 | * can be detected. |
424 | */ |
425 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 0); |
426 | WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); |
427 | out: |
428 | return (wptr & ih->ptr_mask); |
429 | } |
430 | |
431 | /** |
432 | * ih_v6_0_irq_rearm - rearm IRQ if lost |
433 | * |
434 | * @adev: amdgpu_device pointer |
435 | * @ih: amdgpu_ih_ring pointer |
436 | * |
437 | */ |
438 | static void ih_v6_0_irq_rearm(struct amdgpu_device *adev, |
439 | struct amdgpu_ih_ring *ih) |
440 | { |
441 | uint32_t v = 0; |
442 | uint32_t i = 0; |
443 | struct amdgpu_ih_regs *ih_regs; |
444 | |
445 | ih_regs = &ih->ih_regs; |
446 | |
447 | /* Rearm IRQ / re-write doorbell if doorbell write is lost */ |
448 | for (i = 0; i < MAX_REARM_RETRY; i++) { |
449 | v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr); |
450 | if ((v < ih->ring_size) && (v != ih->rptr)) |
451 | WDOORBELL32(ih->doorbell_index, ih->rptr); |
452 | else |
453 | break; |
454 | } |
455 | } |
456 | |
457 | /** |
458 | * ih_v6_0_set_rptr - set the IH ring buffer rptr |
459 | * |
460 | * @adev: amdgpu_device pointer |
461 | * @ih: amdgpu_ih_ring pointer |
462 | * |
463 | * Set the IH ring buffer rptr. |
464 | */ |
465 | static void ih_v6_0_set_rptr(struct amdgpu_device *adev, |
466 | struct amdgpu_ih_ring *ih) |
467 | { |
468 | struct amdgpu_ih_regs *ih_regs; |
469 | |
470 | if (ih->use_doorbell) { |
471 | /* XXX check if swapping is necessary on BE */ |
472 | *ih->rptr_cpu = ih->rptr; |
473 | WDOORBELL32(ih->doorbell_index, ih->rptr); |
474 | |
475 | if (amdgpu_sriov_vf(adev)) |
476 | ih_v6_0_irq_rearm(adev, ih); |
477 | } else { |
478 | ih_regs = &ih->ih_regs; |
479 | WREG32(ih_regs->ih_rb_rptr, ih->rptr); |
480 | } |
481 | } |
482 | |
483 | /** |
484 | * ih_v6_0_self_irq - dispatch work for ring 1 |
485 | * |
486 | * @adev: amdgpu_device pointer |
487 | * @source: irq source |
488 | * @entry: IV with WPTR update |
489 | * |
490 | * Update the WPTR from the IV and schedule work to handle the entries. |
491 | */ |
492 | static int ih_v6_0_self_irq(struct amdgpu_device *adev, |
493 | struct amdgpu_irq_src *source, |
494 | struct amdgpu_iv_entry *entry) |
495 | { |
496 | uint32_t wptr = cpu_to_le32(entry->src_data[0]); |
497 | |
498 | switch (entry->ring_id) { |
499 | case 1: |
500 | *adev->irq.ih1.wptr_cpu = wptr; |
501 | schedule_work(work: &adev->irq.ih1_work); |
502 | break; |
503 | default: |
504 | break; |
505 | } |
506 | return 0; |
507 | } |
508 | |
509 | static const struct amdgpu_irq_src_funcs ih_v6_0_self_irq_funcs = { |
510 | .process = ih_v6_0_self_irq, |
511 | }; |
512 | |
513 | static void ih_v6_0_set_self_irq_funcs(struct amdgpu_device *adev) |
514 | { |
515 | adev->irq.self_irq.num_types = 0; |
516 | adev->irq.self_irq.funcs = &ih_v6_0_self_irq_funcs; |
517 | } |
518 | |
519 | static int ih_v6_0_early_init(void *handle) |
520 | { |
521 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
522 | |
523 | ih_v6_0_set_interrupt_funcs(adev); |
524 | ih_v6_0_set_self_irq_funcs(adev); |
525 | return 0; |
526 | } |
527 | |
528 | static int ih_v6_0_sw_init(void *handle) |
529 | { |
530 | int r; |
531 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
532 | bool use_bus_addr; |
533 | |
534 | r = amdgpu_irq_add_id(adev, client_id: SOC21_IH_CLIENTID_IH, src_id: 0, |
535 | source: &adev->irq.self_irq); |
536 | |
537 | if (r) |
538 | return r; |
539 | |
540 | /* use gpu virtual address for ih ring |
541 | * until ih_checken is programmed to allow |
542 | * use bus address for ih ring by psp bl */ |
543 | use_bus_addr = |
544 | (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) ? false : true; |
545 | r = amdgpu_ih_ring_init(adev, ih: &adev->irq.ih, IH_RING_SIZE, use_bus_addr); |
546 | if (r) |
547 | return r; |
548 | |
549 | adev->irq.ih.use_doorbell = true; |
550 | adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; |
551 | |
552 | adev->irq.ih1.ring_size = 0; |
553 | adev->irq.ih2.ring_size = 0; |
554 | |
555 | /* initialize ih control register offset */ |
556 | ih_v6_0_init_register_offset(adev); |
557 | |
558 | r = amdgpu_ih_ring_init(adev, ih: &adev->irq.ih_soft, IH_SW_RING_SIZE, use_bus_addr: true); |
559 | if (r) |
560 | return r; |
561 | |
562 | r = amdgpu_irq_init(adev); |
563 | |
564 | return r; |
565 | } |
566 | |
567 | static int ih_v6_0_sw_fini(void *handle) |
568 | { |
569 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
570 | |
571 | amdgpu_irq_fini_sw(adev); |
572 | |
573 | return 0; |
574 | } |
575 | |
576 | static int ih_v6_0_hw_init(void *handle) |
577 | { |
578 | int r; |
579 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
580 | |
581 | r = ih_v6_0_irq_init(adev); |
582 | if (r) |
583 | return r; |
584 | |
585 | return 0; |
586 | } |
587 | |
588 | static int ih_v6_0_hw_fini(void *handle) |
589 | { |
590 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
591 | |
592 | ih_v6_0_irq_disable(adev); |
593 | |
594 | return 0; |
595 | } |
596 | |
597 | static int ih_v6_0_suspend(void *handle) |
598 | { |
599 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
600 | |
601 | return ih_v6_0_hw_fini(handle: adev); |
602 | } |
603 | |
604 | static int ih_v6_0_resume(void *handle) |
605 | { |
606 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
607 | |
608 | return ih_v6_0_hw_init(handle: adev); |
609 | } |
610 | |
611 | static bool ih_v6_0_is_idle(void *handle) |
612 | { |
613 | /* todo */ |
614 | return true; |
615 | } |
616 | |
617 | static int ih_v6_0_wait_for_idle(void *handle) |
618 | { |
619 | /* todo */ |
620 | return -ETIMEDOUT; |
621 | } |
622 | |
623 | static int ih_v6_0_soft_reset(void *handle) |
624 | { |
625 | /* todo */ |
626 | return 0; |
627 | } |
628 | |
629 | static void ih_v6_0_update_clockgating_state(struct amdgpu_device *adev, |
630 | bool enable) |
631 | { |
632 | uint32_t data, def, field_val; |
633 | |
634 | if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) { |
635 | def = data = RREG32_SOC15(OSSSYS, 0, regIH_CLK_CTRL); |
636 | field_val = enable ? 0 : 1; |
637 | data = REG_SET_FIELD(data, IH_CLK_CTRL, |
638 | DBUS_MUX_CLK_SOFT_OVERRIDE, field_val); |
639 | data = REG_SET_FIELD(data, IH_CLK_CTRL, |
640 | OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val); |
641 | data = REG_SET_FIELD(data, IH_CLK_CTRL, |
642 | LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val); |
643 | data = REG_SET_FIELD(data, IH_CLK_CTRL, |
644 | DYN_CLK_SOFT_OVERRIDE, field_val); |
645 | data = REG_SET_FIELD(data, IH_CLK_CTRL, |
646 | REG_CLK_SOFT_OVERRIDE, field_val); |
647 | if (def != data) |
648 | WREG32_SOC15(OSSSYS, 0, regIH_CLK_CTRL, data); |
649 | } |
650 | } |
651 | |
652 | static int ih_v6_0_set_clockgating_state(void *handle, |
653 | enum amd_clockgating_state state) |
654 | { |
655 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
656 | |
657 | ih_v6_0_update_clockgating_state(adev, |
658 | enable: state == AMD_CG_STATE_GATE); |
659 | return 0; |
660 | } |
661 | |
662 | static void ih_v6_0_update_ih_mem_power_gating(struct amdgpu_device *adev, |
663 | bool enable) |
664 | { |
665 | uint32_t ih_mem_pwr_cntl; |
666 | |
667 | /* Disable ih sram power cntl before switch powergating mode */ |
668 | ih_mem_pwr_cntl = RREG32_SOC15(OSSSYS, 0, regIH_MEM_POWER_CTRL); |
669 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
670 | IH_BUFFER_MEM_POWER_CTRL_EN, 0); |
671 | WREG32_SOC15(OSSSYS, 0, regIH_MEM_POWER_CTRL, ih_mem_pwr_cntl); |
672 | |
673 | /* It is recommended to set mem powergating mode to DS mode */ |
674 | if (enable) { |
675 | /* mem power mode */ |
676 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
677 | IH_BUFFER_MEM_POWER_LS_EN, 0); |
678 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
679 | IH_BUFFER_MEM_POWER_DS_EN, 1); |
680 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
681 | IH_BUFFER_MEM_POWER_SD_EN, 0); |
682 | /* cam mem power mode */ |
683 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
684 | IH_RETRY_INT_CAM_MEM_POWER_LS_EN, 0); |
685 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
686 | IH_RETRY_INT_CAM_MEM_POWER_DS_EN, 1); |
687 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
688 | IH_RETRY_INT_CAM_MEM_POWER_SD_EN, 0); |
689 | /* re-enable power cntl */ |
690 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
691 | IH_BUFFER_MEM_POWER_CTRL_EN, 1); |
692 | } else { |
693 | /* mem power mode */ |
694 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
695 | IH_BUFFER_MEM_POWER_LS_EN, 0); |
696 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
697 | IH_BUFFER_MEM_POWER_DS_EN, 0); |
698 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
699 | IH_BUFFER_MEM_POWER_SD_EN, 0); |
700 | /* cam mem power mode */ |
701 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
702 | IH_RETRY_INT_CAM_MEM_POWER_LS_EN, 0); |
703 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
704 | IH_RETRY_INT_CAM_MEM_POWER_DS_EN, 0); |
705 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
706 | IH_RETRY_INT_CAM_MEM_POWER_SD_EN, 0); |
707 | /* re-enable power cntl*/ |
708 | ih_mem_pwr_cntl = REG_SET_FIELD(ih_mem_pwr_cntl, IH_MEM_POWER_CTRL, |
709 | IH_BUFFER_MEM_POWER_CTRL_EN, 1); |
710 | } |
711 | |
712 | WREG32_SOC15(OSSSYS, 0, regIH_MEM_POWER_CTRL, ih_mem_pwr_cntl); |
713 | } |
714 | |
715 | static int ih_v6_0_set_powergating_state(void *handle, |
716 | enum amd_powergating_state state) |
717 | { |
718 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
719 | bool enable = (state == AMD_PG_STATE_GATE); |
720 | |
721 | if (adev->pg_flags & AMD_PG_SUPPORT_IH_SRAM_PG) |
722 | ih_v6_0_update_ih_mem_power_gating(adev, enable); |
723 | |
724 | return 0; |
725 | } |
726 | |
727 | static void ih_v6_0_get_clockgating_state(void *handle, u64 *flags) |
728 | { |
729 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
730 | |
731 | if (!RREG32_SOC15(OSSSYS, 0, regIH_CLK_CTRL)) |
732 | *flags |= AMD_CG_SUPPORT_IH_CG; |
733 | } |
734 | |
735 | static const struct amd_ip_funcs ih_v6_0_ip_funcs = { |
736 | .name = "ih_v6_0" , |
737 | .early_init = ih_v6_0_early_init, |
738 | .late_init = NULL, |
739 | .sw_init = ih_v6_0_sw_init, |
740 | .sw_fini = ih_v6_0_sw_fini, |
741 | .hw_init = ih_v6_0_hw_init, |
742 | .hw_fini = ih_v6_0_hw_fini, |
743 | .suspend = ih_v6_0_suspend, |
744 | .resume = ih_v6_0_resume, |
745 | .is_idle = ih_v6_0_is_idle, |
746 | .wait_for_idle = ih_v6_0_wait_for_idle, |
747 | .soft_reset = ih_v6_0_soft_reset, |
748 | .set_clockgating_state = ih_v6_0_set_clockgating_state, |
749 | .set_powergating_state = ih_v6_0_set_powergating_state, |
750 | .get_clockgating_state = ih_v6_0_get_clockgating_state, |
751 | }; |
752 | |
753 | static const struct amdgpu_ih_funcs ih_v6_0_funcs = { |
754 | .get_wptr = ih_v6_0_get_wptr, |
755 | .decode_iv = amdgpu_ih_decode_iv_helper, |
756 | .decode_iv_ts = amdgpu_ih_decode_iv_ts_helper, |
757 | .set_rptr = ih_v6_0_set_rptr |
758 | }; |
759 | |
760 | static void ih_v6_0_set_interrupt_funcs(struct amdgpu_device *adev) |
761 | { |
762 | adev->irq.ih_funcs = &ih_v6_0_funcs; |
763 | } |
764 | |
765 | const struct amdgpu_ip_block_version ih_v6_0_ip_block = { |
766 | .type = AMD_IP_BLOCK_TYPE_IH, |
767 | .major = 6, |
768 | .minor = 0, |
769 | .rev = 0, |
770 | .funcs = &ih_v6_0_ip_funcs, |
771 | }; |
772 | |