1 | /* |
2 | * Copyright 2012 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | |
24 | #include <linux/pci.h> |
25 | |
26 | #include "amdgpu.h" |
27 | #include "amdgpu_ih.h" |
28 | #include "cikd.h" |
29 | |
30 | #include "bif/bif_4_1_d.h" |
31 | #include "bif/bif_4_1_sh_mask.h" |
32 | |
33 | #include "oss/oss_2_0_d.h" |
34 | #include "oss/oss_2_0_sh_mask.h" |
35 | |
36 | /* |
37 | * Interrupts |
38 | * Starting with r6xx, interrupts are handled via a ring buffer. |
39 | * Ring buffers are areas of GPU accessible memory that the GPU |
40 | * writes interrupt vectors into and the host reads vectors out of. |
41 | * There is a rptr (read pointer) that determines where the |
42 | * host is currently reading, and a wptr (write pointer) |
43 | * which determines where the GPU has written. When the |
44 | * pointers are equal, the ring is idle. When the GPU |
45 | * writes vectors to the ring buffer, it increments the |
46 | * wptr. When there is an interrupt, the host then starts |
47 | * fetching commands and processing them until the pointers are |
48 | * equal again at which point it updates the rptr. |
49 | */ |
50 | |
51 | static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev); |
52 | |
53 | /** |
54 | * cik_ih_enable_interrupts - Enable the interrupt ring buffer |
55 | * |
56 | * @adev: amdgpu_device pointer |
57 | * |
58 | * Enable the interrupt ring buffer (CIK). |
59 | */ |
60 | static void cik_ih_enable_interrupts(struct amdgpu_device *adev) |
61 | { |
62 | u32 ih_cntl = RREG32(mmIH_CNTL); |
63 | u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL); |
64 | |
65 | ih_cntl |= IH_CNTL__ENABLE_INTR_MASK; |
66 | ih_rb_cntl |= IH_RB_CNTL__RB_ENABLE_MASK; |
67 | WREG32(mmIH_CNTL, ih_cntl); |
68 | WREG32(mmIH_RB_CNTL, ih_rb_cntl); |
69 | adev->irq.ih.enabled = true; |
70 | } |
71 | |
72 | /** |
73 | * cik_ih_disable_interrupts - Disable the interrupt ring buffer |
74 | * |
75 | * @adev: amdgpu_device pointer |
76 | * |
77 | * Disable the interrupt ring buffer (CIK). |
78 | */ |
79 | static void cik_ih_disable_interrupts(struct amdgpu_device *adev) |
80 | { |
81 | u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL); |
82 | u32 ih_cntl = RREG32(mmIH_CNTL); |
83 | |
84 | ih_rb_cntl &= ~IH_RB_CNTL__RB_ENABLE_MASK; |
85 | ih_cntl &= ~IH_CNTL__ENABLE_INTR_MASK; |
86 | WREG32(mmIH_RB_CNTL, ih_rb_cntl); |
87 | WREG32(mmIH_CNTL, ih_cntl); |
88 | /* set rptr, wptr to 0 */ |
89 | WREG32(mmIH_RB_RPTR, 0); |
90 | WREG32(mmIH_RB_WPTR, 0); |
91 | adev->irq.ih.enabled = false; |
92 | adev->irq.ih.rptr = 0; |
93 | } |
94 | |
95 | /** |
96 | * cik_ih_irq_init - init and enable the interrupt ring |
97 | * |
98 | * @adev: amdgpu_device pointer |
99 | * |
100 | * Allocate a ring buffer for the interrupt controller, |
101 | * enable the RLC, disable interrupts, enable the IH |
102 | * ring buffer and enable it (CIK). |
103 | * Called at device load and reume. |
104 | * Returns 0 for success, errors for failure. |
105 | */ |
106 | static int cik_ih_irq_init(struct amdgpu_device *adev) |
107 | { |
108 | struct amdgpu_ih_ring *ih = &adev->irq.ih; |
109 | int rb_bufsz; |
110 | u32 interrupt_cntl, ih_cntl, ih_rb_cntl; |
111 | |
112 | /* disable irqs */ |
113 | cik_ih_disable_interrupts(adev); |
114 | |
115 | /* setup interrupt control */ |
116 | WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8); |
117 | interrupt_cntl = RREG32(mmINTERRUPT_CNTL); |
118 | /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi |
119 | * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN |
120 | */ |
121 | interrupt_cntl &= ~INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK; |
122 | /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */ |
123 | interrupt_cntl &= ~INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK; |
124 | WREG32(mmINTERRUPT_CNTL, interrupt_cntl); |
125 | |
126 | WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8); |
127 | rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4); |
128 | |
129 | ih_rb_cntl = (IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK | |
130 | IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK | |
131 | (rb_bufsz << 1)); |
132 | |
133 | ih_rb_cntl |= IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK; |
134 | |
135 | /* set the writeback address whether it's enabled or not */ |
136 | WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr)); |
137 | WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF); |
138 | |
139 | WREG32(mmIH_RB_CNTL, ih_rb_cntl); |
140 | |
141 | /* set rptr, wptr to 0 */ |
142 | WREG32(mmIH_RB_RPTR, 0); |
143 | WREG32(mmIH_RB_WPTR, 0); |
144 | |
145 | /* Default settings for IH_CNTL (disabled at first) */ |
146 | ih_cntl = (0x10 << IH_CNTL__MC_WRREQ_CREDIT__SHIFT) | |
147 | (0x10 << IH_CNTL__MC_WR_CLEAN_CNT__SHIFT) | |
148 | (0 << IH_CNTL__MC_VMID__SHIFT); |
149 | /* IH_CNTL__RPTR_REARM_MASK only works if msi's are enabled */ |
150 | if (adev->irq.msi_enabled) |
151 | ih_cntl |= IH_CNTL__RPTR_REARM_MASK; |
152 | WREG32(mmIH_CNTL, ih_cntl); |
153 | |
154 | pci_set_master(dev: adev->pdev); |
155 | |
156 | /* enable irqs */ |
157 | cik_ih_enable_interrupts(adev); |
158 | |
159 | return 0; |
160 | } |
161 | |
162 | /** |
163 | * cik_ih_irq_disable - disable interrupts |
164 | * |
165 | * @adev: amdgpu_device pointer |
166 | * |
167 | * Disable interrupts on the hw (CIK). |
168 | */ |
169 | static void cik_ih_irq_disable(struct amdgpu_device *adev) |
170 | { |
171 | cik_ih_disable_interrupts(adev); |
172 | /* Wait and acknowledge irq */ |
173 | mdelay(1); |
174 | } |
175 | |
176 | /** |
177 | * cik_ih_get_wptr - get the IH ring buffer wptr |
178 | * |
179 | * @adev: amdgpu_device pointer |
180 | * @ih: IH ring buffer to fetch wptr |
181 | * |
182 | * Get the IH ring buffer wptr from either the register |
183 | * or the writeback memory buffer (CIK). Also check for |
184 | * ring buffer overflow and deal with it. |
185 | * Used by cik_irq_process(). |
186 | * Returns the value of the wptr. |
187 | */ |
188 | static u32 cik_ih_get_wptr(struct amdgpu_device *adev, |
189 | struct amdgpu_ih_ring *ih) |
190 | { |
191 | u32 wptr, tmp; |
192 | |
193 | wptr = le32_to_cpu(*ih->wptr_cpu); |
194 | |
195 | if (wptr & IH_RB_WPTR__RB_OVERFLOW_MASK) { |
196 | wptr &= ~IH_RB_WPTR__RB_OVERFLOW_MASK; |
197 | /* When a ring buffer overflow happen start parsing interrupt |
198 | * from the last not overwritten vector (wptr + 16). Hopefully |
199 | * this should allow us to catchup. |
200 | */ |
201 | dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n" , |
202 | wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); |
203 | ih->rptr = (wptr + 16) & ih->ptr_mask; |
204 | tmp = RREG32(mmIH_RB_CNTL); |
205 | tmp |= IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK; |
206 | WREG32(mmIH_RB_CNTL, tmp); |
207 | |
208 | /* Unset the CLEAR_OVERFLOW bit immediately so new overflows |
209 | * can be detected. |
210 | */ |
211 | tmp &= ~IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK; |
212 | WREG32(mmIH_RB_CNTL, tmp); |
213 | } |
214 | return (wptr & ih->ptr_mask); |
215 | } |
216 | |
217 | /* CIK IV Ring |
218 | * Each IV ring entry is 128 bits: |
219 | * [7:0] - interrupt source id |
220 | * [31:8] - reserved |
221 | * [59:32] - interrupt source data |
222 | * [63:60] - reserved |
223 | * [71:64] - RINGID |
224 | * CP: |
225 | * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0] |
226 | * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher |
227 | * - for gfx, hw shader state (0=PS...5=LS, 6=CS) |
228 | * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes |
229 | * PIPE_ID - ME0 0=3D |
230 | * - ME1&2 compute dispatcher (4 pipes each) |
231 | * SDMA: |
232 | * INSTANCE_ID [1:0], QUEUE_ID[1:0] |
233 | * INSTANCE_ID - 0 = sdma0, 1 = sdma1 |
234 | * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1 |
235 | * [79:72] - VMID |
236 | * [95:80] - PASID |
237 | * [127:96] - reserved |
238 | */ |
239 | |
240 | /** |
241 | * cik_ih_decode_iv - decode an interrupt vector |
242 | * |
243 | * @adev: amdgpu_device pointer |
244 | * |
245 | * Decodes the interrupt vector at the current rptr |
246 | * position and also advance the position. |
247 | */ |
248 | static void cik_ih_decode_iv(struct amdgpu_device *adev, |
249 | struct amdgpu_ih_ring *ih, |
250 | struct amdgpu_iv_entry *entry) |
251 | { |
252 | /* wptr/rptr are in bytes! */ |
253 | u32 ring_index = ih->rptr >> 2; |
254 | uint32_t dw[4]; |
255 | |
256 | dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); |
257 | dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); |
258 | dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); |
259 | dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); |
260 | |
261 | entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY; |
262 | entry->src_id = dw[0] & 0xff; |
263 | entry->src_data[0] = dw[1] & 0xfffffff; |
264 | entry->ring_id = dw[2] & 0xff; |
265 | entry->vmid = (dw[2] >> 8) & 0xff; |
266 | entry->pasid = (dw[2] >> 16) & 0xffff; |
267 | |
268 | /* wptr/rptr are in bytes! */ |
269 | ih->rptr += 16; |
270 | } |
271 | |
272 | /** |
273 | * cik_ih_set_rptr - set the IH ring buffer rptr |
274 | * |
275 | * @adev: amdgpu_device pointer |
276 | * @ih: IH ring buffer to set wptr |
277 | * |
278 | * Set the IH ring buffer rptr. |
279 | */ |
280 | static void cik_ih_set_rptr(struct amdgpu_device *adev, |
281 | struct amdgpu_ih_ring *ih) |
282 | { |
283 | WREG32(mmIH_RB_RPTR, ih->rptr); |
284 | } |
285 | |
286 | static int cik_ih_early_init(void *handle) |
287 | { |
288 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
289 | int ret; |
290 | |
291 | ret = amdgpu_irq_add_domain(adev); |
292 | if (ret) |
293 | return ret; |
294 | |
295 | cik_ih_set_interrupt_funcs(adev); |
296 | |
297 | return 0; |
298 | } |
299 | |
300 | static int cik_ih_sw_init(void *handle) |
301 | { |
302 | int r; |
303 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
304 | |
305 | r = amdgpu_ih_ring_init(adev, ih: &adev->irq.ih, ring_size: 64 * 1024, use_bus_addr: false); |
306 | if (r) |
307 | return r; |
308 | |
309 | r = amdgpu_irq_init(adev); |
310 | |
311 | return r; |
312 | } |
313 | |
314 | static int cik_ih_sw_fini(void *handle) |
315 | { |
316 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
317 | |
318 | amdgpu_irq_fini_sw(adev); |
319 | amdgpu_irq_remove_domain(adev); |
320 | |
321 | return 0; |
322 | } |
323 | |
324 | static int cik_ih_hw_init(void *handle) |
325 | { |
326 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
327 | |
328 | return cik_ih_irq_init(adev); |
329 | } |
330 | |
331 | static int cik_ih_hw_fini(void *handle) |
332 | { |
333 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
334 | |
335 | cik_ih_irq_disable(adev); |
336 | |
337 | return 0; |
338 | } |
339 | |
340 | static int cik_ih_suspend(void *handle) |
341 | { |
342 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
343 | |
344 | return cik_ih_hw_fini(handle: adev); |
345 | } |
346 | |
347 | static int cik_ih_resume(void *handle) |
348 | { |
349 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
350 | |
351 | return cik_ih_hw_init(handle: adev); |
352 | } |
353 | |
354 | static bool cik_ih_is_idle(void *handle) |
355 | { |
356 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
357 | u32 tmp = RREG32(mmSRBM_STATUS); |
358 | |
359 | if (tmp & SRBM_STATUS__IH_BUSY_MASK) |
360 | return false; |
361 | |
362 | return true; |
363 | } |
364 | |
365 | static int cik_ih_wait_for_idle(void *handle) |
366 | { |
367 | unsigned i; |
368 | u32 tmp; |
369 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
370 | |
371 | for (i = 0; i < adev->usec_timeout; i++) { |
372 | /* read MC_STATUS */ |
373 | tmp = RREG32(mmSRBM_STATUS) & SRBM_STATUS__IH_BUSY_MASK; |
374 | if (!tmp) |
375 | return 0; |
376 | udelay(1); |
377 | } |
378 | return -ETIMEDOUT; |
379 | } |
380 | |
381 | static int cik_ih_soft_reset(void *handle) |
382 | { |
383 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
384 | |
385 | u32 srbm_soft_reset = 0; |
386 | u32 tmp = RREG32(mmSRBM_STATUS); |
387 | |
388 | if (tmp & SRBM_STATUS__IH_BUSY_MASK) |
389 | srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK; |
390 | |
391 | if (srbm_soft_reset) { |
392 | tmp = RREG32(mmSRBM_SOFT_RESET); |
393 | tmp |= srbm_soft_reset; |
394 | dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n" , tmp); |
395 | WREG32(mmSRBM_SOFT_RESET, tmp); |
396 | tmp = RREG32(mmSRBM_SOFT_RESET); |
397 | |
398 | udelay(50); |
399 | |
400 | tmp &= ~srbm_soft_reset; |
401 | WREG32(mmSRBM_SOFT_RESET, tmp); |
402 | tmp = RREG32(mmSRBM_SOFT_RESET); |
403 | |
404 | /* Wait a little for things to settle down */ |
405 | udelay(50); |
406 | } |
407 | |
408 | return 0; |
409 | } |
410 | |
411 | static int cik_ih_set_clockgating_state(void *handle, |
412 | enum amd_clockgating_state state) |
413 | { |
414 | return 0; |
415 | } |
416 | |
417 | static int cik_ih_set_powergating_state(void *handle, |
418 | enum amd_powergating_state state) |
419 | { |
420 | return 0; |
421 | } |
422 | |
423 | static const struct amd_ip_funcs cik_ih_ip_funcs = { |
424 | .name = "cik_ih" , |
425 | .early_init = cik_ih_early_init, |
426 | .late_init = NULL, |
427 | .sw_init = cik_ih_sw_init, |
428 | .sw_fini = cik_ih_sw_fini, |
429 | .hw_init = cik_ih_hw_init, |
430 | .hw_fini = cik_ih_hw_fini, |
431 | .suspend = cik_ih_suspend, |
432 | .resume = cik_ih_resume, |
433 | .is_idle = cik_ih_is_idle, |
434 | .wait_for_idle = cik_ih_wait_for_idle, |
435 | .soft_reset = cik_ih_soft_reset, |
436 | .set_clockgating_state = cik_ih_set_clockgating_state, |
437 | .set_powergating_state = cik_ih_set_powergating_state, |
438 | }; |
439 | |
440 | static const struct amdgpu_ih_funcs cik_ih_funcs = { |
441 | .get_wptr = cik_ih_get_wptr, |
442 | .decode_iv = cik_ih_decode_iv, |
443 | .set_rptr = cik_ih_set_rptr |
444 | }; |
445 | |
446 | static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev) |
447 | { |
448 | adev->irq.ih_funcs = &cik_ih_funcs; |
449 | } |
450 | |
451 | const struct amdgpu_ip_block_version cik_ih_ip_block = { |
452 | .type = AMD_IP_BLOCK_TYPE_IH, |
453 | .major = 2, |
454 | .minor = 0, |
455 | .rev = 0, |
456 | .funcs = &cik_ih_ip_funcs, |
457 | }; |
458 | |