1 | /* |
2 | * Copyright 2022 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | #include "amdgpu.h" |
24 | #include "amdgpu_atombios.h" |
25 | #include "nbio_v7_9.h" |
26 | #include "amdgpu_ras.h" |
27 | |
28 | #include "nbio/nbio_7_9_0_offset.h" |
29 | #include "nbio/nbio_7_9_0_sh_mask.h" |
30 | #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" |
31 | #include <uapi/linux/kfd_ioctl.h> |
32 | |
33 | #define NPS_MODE_MASK 0x000000FFL |
34 | |
35 | /* Core 0 Port 0 counter */ |
36 | #define smnPCIEP_NAK_COUNTER 0x1A340218 |
37 | |
38 | static void nbio_v7_9_remap_hdp_registers(struct amdgpu_device *adev) |
39 | { |
40 | WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL, |
41 | adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); |
42 | WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL, |
43 | adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); |
44 | } |
45 | |
46 | static u32 nbio_v7_9_get_rev_id(struct amdgpu_device *adev) |
47 | { |
48 | u32 tmp; |
49 | |
50 | tmp = IP_VERSION_SUBREV(amdgpu_ip_version_full(adev, NBIO_HWIP, 0)); |
51 | /* If it is VF or subrevision holds a non-zero value, that should be used */ |
52 | if (tmp || amdgpu_sriov_vf(adev)) |
53 | return tmp; |
54 | |
55 | /* If discovery subrev is not updated, use register version */ |
56 | tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0); |
57 | tmp = REG_GET_FIELD(tmp, RCC_STRAP0_RCC_DEV0_EPF0_STRAP0, |
58 | STRAP_ATI_REV_ID_DEV0_F0); |
59 | |
60 | return tmp; |
61 | } |
62 | |
63 | static void nbio_v7_9_mc_access_enable(struct amdgpu_device *adev, bool enable) |
64 | { |
65 | if (enable) |
66 | WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, |
67 | BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK | BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK); |
68 | else |
69 | WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0); |
70 | } |
71 | |
72 | static u32 nbio_v7_9_get_memsize(struct amdgpu_device *adev) |
73 | { |
74 | return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE); |
75 | } |
76 | |
77 | static void nbio_v7_9_sdma_doorbell_range(struct amdgpu_device *adev, int instance, |
78 | bool use_doorbell, int doorbell_index, int doorbell_size) |
79 | { |
80 | u32 doorbell_range = 0, doorbell_ctrl = 0; |
81 | int aid_id, dev_inst; |
82 | |
83 | dev_inst = GET_INST(SDMA0, instance); |
84 | aid_id = adev->sdma.instance[instance].aid_id; |
85 | |
86 | if (use_doorbell == false) |
87 | return; |
88 | |
89 | doorbell_range = |
90 | REG_SET_FIELD(doorbell_range, DOORBELL0_CTRL_ENTRY_0, |
91 | BIF_DOORBELL0_RANGE_OFFSET_ENTRY, doorbell_index); |
92 | doorbell_range = |
93 | REG_SET_FIELD(doorbell_range, DOORBELL0_CTRL_ENTRY_0, |
94 | BIF_DOORBELL0_RANGE_SIZE_ENTRY, doorbell_size); |
95 | doorbell_ctrl = |
96 | REG_SET_FIELD(doorbell_ctrl, S2A_DOORBELL_ENTRY_1_CTRL, |
97 | S2A_DOORBELL_PORT1_ENABLE, 1); |
98 | doorbell_ctrl = |
99 | REG_SET_FIELD(doorbell_ctrl, S2A_DOORBELL_ENTRY_1_CTRL, |
100 | S2A_DOORBELL_PORT1_RANGE_SIZE, doorbell_size); |
101 | |
102 | switch (dev_inst % adev->sdma.num_inst_per_aid) { |
103 | case 0: |
104 | WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_1, |
105 | 4 * aid_id, doorbell_range); |
106 | |
107 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
108 | S2A_DOORBELL_ENTRY_1_CTRL, |
109 | S2A_DOORBELL_PORT1_AWID, 0xe); |
110 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
111 | S2A_DOORBELL_ENTRY_1_CTRL, |
112 | S2A_DOORBELL_PORT1_RANGE_OFFSET, 0xe); |
113 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
114 | S2A_DOORBELL_ENTRY_1_CTRL, |
115 | S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, |
116 | 0x1); |
117 | WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_1_CTRL, |
118 | aid_id, doorbell_ctrl); |
119 | break; |
120 | case 1: |
121 | WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_2, |
122 | 4 * aid_id, doorbell_range); |
123 | |
124 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
125 | S2A_DOORBELL_ENTRY_1_CTRL, |
126 | S2A_DOORBELL_PORT1_AWID, 0x8); |
127 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
128 | S2A_DOORBELL_ENTRY_1_CTRL, |
129 | S2A_DOORBELL_PORT1_RANGE_OFFSET, 0x8); |
130 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
131 | S2A_DOORBELL_ENTRY_1_CTRL, |
132 | S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, |
133 | 0x2); |
134 | WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_2_CTRL, |
135 | aid_id, doorbell_ctrl); |
136 | break; |
137 | case 2: |
138 | WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_3, |
139 | 4 * aid_id, doorbell_range); |
140 | |
141 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
142 | S2A_DOORBELL_ENTRY_1_CTRL, |
143 | S2A_DOORBELL_PORT1_AWID, 0x9); |
144 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
145 | S2A_DOORBELL_ENTRY_1_CTRL, |
146 | S2A_DOORBELL_PORT1_RANGE_OFFSET, 0x9); |
147 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
148 | S2A_DOORBELL_ENTRY_1_CTRL, |
149 | S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, |
150 | 0x8); |
151 | WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_5_CTRL, |
152 | aid_id, doorbell_ctrl); |
153 | break; |
154 | case 3: |
155 | WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_4, |
156 | 4 * aid_id, doorbell_range); |
157 | |
158 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
159 | S2A_DOORBELL_ENTRY_1_CTRL, |
160 | S2A_DOORBELL_PORT1_AWID, 0xa); |
161 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
162 | S2A_DOORBELL_ENTRY_1_CTRL, |
163 | S2A_DOORBELL_PORT1_RANGE_OFFSET, 0xa); |
164 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
165 | S2A_DOORBELL_ENTRY_1_CTRL, |
166 | S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, |
167 | 0x9); |
168 | WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_6_CTRL, |
169 | aid_id, doorbell_ctrl); |
170 | break; |
171 | default: |
172 | break; |
173 | } |
174 | } |
175 | |
176 | static void nbio_v7_9_vcn_doorbell_range(struct amdgpu_device *adev, bool use_doorbell, |
177 | int doorbell_index, int instance) |
178 | { |
179 | u32 doorbell_range = 0, doorbell_ctrl = 0; |
180 | u32 aid_id = instance; |
181 | |
182 | if (use_doorbell) { |
183 | doorbell_range = REG_SET_FIELD(doorbell_range, |
184 | DOORBELL0_CTRL_ENTRY_0, |
185 | BIF_DOORBELL0_RANGE_OFFSET_ENTRY, |
186 | doorbell_index); |
187 | doorbell_range = REG_SET_FIELD(doorbell_range, |
188 | DOORBELL0_CTRL_ENTRY_0, |
189 | BIF_DOORBELL0_RANGE_SIZE_ENTRY, |
190 | 0x9); |
191 | if (aid_id) |
192 | doorbell_range = REG_SET_FIELD(doorbell_range, |
193 | DOORBELL0_CTRL_ENTRY_0, |
194 | DOORBELL0_FENCE_ENABLE_ENTRY, |
195 | 0x4); |
196 | |
197 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
198 | S2A_DOORBELL_ENTRY_1_CTRL, |
199 | S2A_DOORBELL_PORT1_ENABLE, 1); |
200 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
201 | S2A_DOORBELL_ENTRY_1_CTRL, |
202 | S2A_DOORBELL_PORT1_AWID, 0x4); |
203 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
204 | S2A_DOORBELL_ENTRY_1_CTRL, |
205 | S2A_DOORBELL_PORT1_RANGE_OFFSET, 0x4); |
206 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
207 | S2A_DOORBELL_ENTRY_1_CTRL, |
208 | S2A_DOORBELL_PORT1_RANGE_SIZE, 0x9); |
209 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
210 | S2A_DOORBELL_ENTRY_1_CTRL, |
211 | S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, 0x4); |
212 | |
213 | WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_17, |
214 | aid_id, doorbell_range); |
215 | WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_4_CTRL, |
216 | aid_id, doorbell_ctrl); |
217 | } else { |
218 | doorbell_range = REG_SET_FIELD(doorbell_range, |
219 | DOORBELL0_CTRL_ENTRY_0, |
220 | BIF_DOORBELL0_RANGE_SIZE_ENTRY, 0); |
221 | doorbell_ctrl = REG_SET_FIELD(doorbell_ctrl, |
222 | S2A_DOORBELL_ENTRY_1_CTRL, |
223 | S2A_DOORBELL_PORT1_RANGE_SIZE, 0); |
224 | |
225 | WREG32_SOC15_OFFSET(NBIO, 0, regDOORBELL0_CTRL_ENTRY_17, |
226 | aid_id, doorbell_range); |
227 | WREG32_SOC15_EXT(NBIO, aid_id, regS2A_DOORBELL_ENTRY_4_CTRL, |
228 | aid_id, doorbell_ctrl); |
229 | } |
230 | } |
231 | |
232 | static void nbio_v7_9_enable_doorbell_aperture(struct amdgpu_device *adev, |
233 | bool enable) |
234 | { |
235 | /* Enable to allow doorbell pass thru on pre-silicon bare-metal */ |
236 | WREG32_SOC15(NBIO, 0, regBIFC_DOORBELL_ACCESS_EN_PF, 0xfffff); |
237 | WREG32_FIELD15_PREREG(NBIO, 0, RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN, |
238 | BIF_DOORBELL_APER_EN, enable ? 1 : 0); |
239 | } |
240 | |
241 | static void nbio_v7_9_enable_doorbell_selfring_aperture(struct amdgpu_device *adev, |
242 | bool enable) |
243 | { |
244 | u32 tmp = 0; |
245 | |
246 | if (enable) { |
247 | tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, |
248 | DOORBELL_SELFRING_GPA_APER_EN, 1) | |
249 | REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, |
250 | DOORBELL_SELFRING_GPA_APER_MODE, 1) | |
251 | REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, |
252 | DOORBELL_SELFRING_GPA_APER_SIZE, 0); |
253 | |
254 | WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW, |
255 | lower_32_bits(adev->doorbell.base)); |
256 | WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH, |
257 | upper_32_bits(adev->doorbell.base)); |
258 | } |
259 | |
260 | WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, tmp); |
261 | } |
262 | |
263 | static void nbio_v7_9_ih_doorbell_range(struct amdgpu_device *adev, |
264 | bool use_doorbell, int doorbell_index) |
265 | { |
266 | u32 ih_doorbell_range = 0, ih_doorbell_ctrl = 0; |
267 | |
268 | if (use_doorbell) { |
269 | ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, |
270 | DOORBELL0_CTRL_ENTRY_0, |
271 | BIF_DOORBELL0_RANGE_OFFSET_ENTRY, |
272 | doorbell_index); |
273 | ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, |
274 | DOORBELL0_CTRL_ENTRY_0, |
275 | BIF_DOORBELL0_RANGE_SIZE_ENTRY, |
276 | 0x8); |
277 | |
278 | ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl, |
279 | S2A_DOORBELL_ENTRY_1_CTRL, |
280 | S2A_DOORBELL_PORT1_ENABLE, 1); |
281 | ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl, |
282 | S2A_DOORBELL_ENTRY_1_CTRL, |
283 | S2A_DOORBELL_PORT1_AWID, 0); |
284 | ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl, |
285 | S2A_DOORBELL_ENTRY_1_CTRL, |
286 | S2A_DOORBELL_PORT1_RANGE_OFFSET, 0); |
287 | ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl, |
288 | S2A_DOORBELL_ENTRY_1_CTRL, |
289 | S2A_DOORBELL_PORT1_RANGE_SIZE, 0x8); |
290 | ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl, |
291 | S2A_DOORBELL_ENTRY_1_CTRL, |
292 | S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE, 0); |
293 | } else { |
294 | ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, |
295 | DOORBELL0_CTRL_ENTRY_0, |
296 | BIF_DOORBELL0_RANGE_SIZE_ENTRY, 0); |
297 | ih_doorbell_ctrl = REG_SET_FIELD(ih_doorbell_ctrl, |
298 | S2A_DOORBELL_ENTRY_1_CTRL, |
299 | S2A_DOORBELL_PORT1_RANGE_SIZE, 0); |
300 | } |
301 | |
302 | WREG32_SOC15(NBIO, 0, regDOORBELL0_CTRL_ENTRY_0, ih_doorbell_range); |
303 | WREG32_SOC15(NBIO, 0, regS2A_DOORBELL_ENTRY_3_CTRL, ih_doorbell_ctrl); |
304 | } |
305 | |
306 | |
307 | static void nbio_v7_9_update_medium_grain_clock_gating(struct amdgpu_device *adev, |
308 | bool enable) |
309 | { |
310 | } |
311 | |
312 | static void nbio_v7_9_update_medium_grain_light_sleep(struct amdgpu_device *adev, |
313 | bool enable) |
314 | { |
315 | } |
316 | |
317 | static void nbio_v7_9_get_clockgating_state(struct amdgpu_device *adev, |
318 | u64 *flags) |
319 | { |
320 | } |
321 | |
322 | static void nbio_v7_9_ih_control(struct amdgpu_device *adev) |
323 | { |
324 | u32 interrupt_cntl; |
325 | |
326 | /* setup interrupt control */ |
327 | WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8); |
328 | interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL); |
329 | /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi |
330 | * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN |
331 | */ |
332 | interrupt_cntl = |
333 | REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0); |
334 | /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */ |
335 | interrupt_cntl = |
336 | REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0); |
337 | WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl); |
338 | } |
339 | |
340 | static u32 nbio_v7_9_get_hdp_flush_req_offset(struct amdgpu_device *adev) |
341 | { |
342 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_REQ); |
343 | } |
344 | |
345 | static u32 nbio_v7_9_get_hdp_flush_done_offset(struct amdgpu_device *adev) |
346 | { |
347 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_DONE); |
348 | } |
349 | |
350 | static u32 nbio_v7_9_get_pcie_index_offset(struct amdgpu_device *adev) |
351 | { |
352 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2); |
353 | } |
354 | |
355 | static u32 nbio_v7_9_get_pcie_data_offset(struct amdgpu_device *adev) |
356 | { |
357 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA2); |
358 | } |
359 | |
360 | static u32 nbio_v7_9_get_pcie_index_hi_offset(struct amdgpu_device *adev) |
361 | { |
362 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2_HI); |
363 | } |
364 | |
365 | const struct nbio_hdp_flush_reg nbio_v7_9_hdp_flush_reg = { |
366 | .ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK, |
367 | .ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK, |
368 | .ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK, |
369 | .ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK, |
370 | .ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK, |
371 | .ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK, |
372 | .ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK, |
373 | .ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK, |
374 | .ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK, |
375 | .ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK, |
376 | .ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK, |
377 | .ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK, |
378 | .ref_and_mask_sdma2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK, |
379 | .ref_and_mask_sdma3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK, |
380 | .ref_and_mask_sdma4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK, |
381 | .ref_and_mask_sdma5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK, |
382 | .ref_and_mask_sdma6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK, |
383 | .ref_and_mask_sdma7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK, |
384 | }; |
385 | |
386 | static void nbio_v7_9_enable_doorbell_interrupt(struct amdgpu_device *adev, |
387 | bool enable) |
388 | { |
389 | WREG32_FIELD15_PREREG(NBIO, 0, BIF_BX0_BIF_DOORBELL_INT_CNTL, |
390 | DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1); |
391 | } |
392 | |
393 | static int nbio_v7_9_get_compute_partition_mode(struct amdgpu_device *adev) |
394 | { |
395 | u32 tmp, px; |
396 | |
397 | tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_COMPUTE_STATUS); |
398 | px = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_COMPUTE_STATUS, |
399 | PARTITION_MODE); |
400 | |
401 | return px; |
402 | } |
403 | |
404 | static u32 nbio_v7_9_get_memory_partition_mode(struct amdgpu_device *adev, |
405 | u32 *supp_modes) |
406 | { |
407 | u32 tmp; |
408 | |
409 | tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_STATUS); |
410 | tmp = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_MEM_STATUS, NPS_MODE); |
411 | |
412 | if (supp_modes) { |
413 | *supp_modes = |
414 | RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_CAP); |
415 | } |
416 | |
417 | return ffs(tmp); |
418 | } |
419 | |
420 | static void nbio_v7_9_init_registers(struct amdgpu_device *adev) |
421 | { |
422 | u32 inst_mask; |
423 | int i; |
424 | |
425 | if (amdgpu_sriov_vf(adev)) |
426 | adev->rmmio_remap.reg_offset = |
427 | SOC15_REG_OFFSET( |
428 | NBIO, 0, |
429 | regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL) |
430 | << 2; |
431 | WREG32_SOC15(NBIO, 0, regXCC_DOORBELL_FENCE, |
432 | 0xff & ~(adev->gfx.xcc_mask)); |
433 | |
434 | WREG32_SOC15(NBIO, 0, regBIFC_GFX_INT_MONITOR_MASK, 0x7ff); |
435 | |
436 | inst_mask = adev->aid_mask & ~1U; |
437 | for_each_inst(i, inst_mask) { |
438 | WREG32_SOC15_EXT(NBIO, i, regXCC_DOORBELL_FENCE, i, |
439 | XCC_DOORBELL_FENCE__SHUB_SLV_MODE_MASK); |
440 | |
441 | } |
442 | |
443 | if (!amdgpu_sriov_vf(adev)) { |
444 | u32 baco_cntl; |
445 | for_each_inst(i, adev->aid_mask) { |
446 | baco_cntl = RREG32_SOC15(NBIO, i, regBIF_BX0_BACO_CNTL); |
447 | if (baco_cntl & (BIF_BX0_BACO_CNTL__BACO_DUMMY_EN_MASK | |
448 | BIF_BX0_BACO_CNTL__BACO_EN_MASK)) { |
449 | baco_cntl &= ~( |
450 | BIF_BX0_BACO_CNTL__BACO_DUMMY_EN_MASK | |
451 | BIF_BX0_BACO_CNTL__BACO_EN_MASK); |
452 | dev_dbg(adev->dev, |
453 | "Unsetting baco dummy mode %x" , |
454 | baco_cntl); |
455 | WREG32_SOC15(NBIO, i, regBIF_BX0_BACO_CNTL, |
456 | baco_cntl); |
457 | } |
458 | } |
459 | } |
460 | } |
461 | |
462 | static u64 nbio_v7_9_get_pcie_replay_count(struct amdgpu_device *adev) |
463 | { |
464 | u32 val, nak_r, nak_g; |
465 | |
466 | if (adev->flags & AMD_IS_APU) |
467 | return 0; |
468 | |
469 | /* Get the number of NAKs received and generated */ |
470 | val = RREG32_PCIE(smnPCIEP_NAK_COUNTER); |
471 | nak_r = val & 0xFFFF; |
472 | nak_g = val >> 16; |
473 | |
474 | /* Add the total number of NAKs, i.e the number of replays */ |
475 | return (nak_r + nak_g); |
476 | } |
477 | |
478 | const struct amdgpu_nbio_funcs nbio_v7_9_funcs = { |
479 | .get_hdp_flush_req_offset = nbio_v7_9_get_hdp_flush_req_offset, |
480 | .get_hdp_flush_done_offset = nbio_v7_9_get_hdp_flush_done_offset, |
481 | .get_pcie_index_offset = nbio_v7_9_get_pcie_index_offset, |
482 | .get_pcie_data_offset = nbio_v7_9_get_pcie_data_offset, |
483 | .get_pcie_index_hi_offset = nbio_v7_9_get_pcie_index_hi_offset, |
484 | .get_rev_id = nbio_v7_9_get_rev_id, |
485 | .mc_access_enable = nbio_v7_9_mc_access_enable, |
486 | .get_memsize = nbio_v7_9_get_memsize, |
487 | .sdma_doorbell_range = nbio_v7_9_sdma_doorbell_range, |
488 | .vcn_doorbell_range = nbio_v7_9_vcn_doorbell_range, |
489 | .enable_doorbell_aperture = nbio_v7_9_enable_doorbell_aperture, |
490 | .enable_doorbell_selfring_aperture = nbio_v7_9_enable_doorbell_selfring_aperture, |
491 | .ih_doorbell_range = nbio_v7_9_ih_doorbell_range, |
492 | .enable_doorbell_interrupt = nbio_v7_9_enable_doorbell_interrupt, |
493 | .update_medium_grain_clock_gating = nbio_v7_9_update_medium_grain_clock_gating, |
494 | .update_medium_grain_light_sleep = nbio_v7_9_update_medium_grain_light_sleep, |
495 | .get_clockgating_state = nbio_v7_9_get_clockgating_state, |
496 | .ih_control = nbio_v7_9_ih_control, |
497 | .remap_hdp_registers = nbio_v7_9_remap_hdp_registers, |
498 | .get_compute_partition_mode = nbio_v7_9_get_compute_partition_mode, |
499 | .get_memory_partition_mode = nbio_v7_9_get_memory_partition_mode, |
500 | .init_registers = nbio_v7_9_init_registers, |
501 | .get_pcie_replay_count = nbio_v7_9_get_pcie_replay_count, |
502 | }; |
503 | |
504 | static void nbio_v7_9_query_ras_error_count(struct amdgpu_device *adev, |
505 | void *ras_error_status) |
506 | { |
507 | } |
508 | |
509 | static void nbio_v7_9_handle_ras_controller_intr_no_bifring(struct amdgpu_device *adev) |
510 | { |
511 | uint32_t bif_doorbell_intr_cntl; |
512 | struct ras_manager *obj = amdgpu_ras_find_obj(adev, head: adev->nbio.ras_if); |
513 | struct ras_err_data err_data; |
514 | struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); |
515 | |
516 | if (amdgpu_ras_error_data_init(err_data: &err_data)) |
517 | return; |
518 | |
519 | bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_DOORBELL_INT_CNTL); |
520 | |
521 | if (REG_GET_FIELD(bif_doorbell_intr_cntl, |
522 | BIF_BX0_BIF_DOORBELL_INT_CNTL, RAS_CNTLR_INTERRUPT_STATUS)) { |
523 | /* driver has to clear the interrupt status when bif ring is disabled */ |
524 | bif_doorbell_intr_cntl = REG_SET_FIELD(bif_doorbell_intr_cntl, |
525 | BIF_BX0_BIF_DOORBELL_INT_CNTL, |
526 | RAS_CNTLR_INTERRUPT_CLEAR, 1); |
527 | WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); |
528 | |
529 | if (!ras->disable_ras_err_cnt_harvest) { |
530 | /* |
531 | * clear error status after ras_controller_intr |
532 | * according to hw team and count ue number |
533 | * for query |
534 | */ |
535 | nbio_v7_9_query_ras_error_count(adev, ras_error_status: &err_data); |
536 | |
537 | /* logging on error cnt and printing for awareness */ |
538 | obj->err_data.ue_count += err_data.ue_count; |
539 | obj->err_data.ce_count += err_data.ce_count; |
540 | |
541 | if (err_data.ce_count) |
542 | dev_info(adev->dev, "%ld correctable hardware " |
543 | "errors detected in %s block\n" , |
544 | obj->err_data.ce_count, |
545 | get_ras_block_str(adev->nbio.ras_if)); |
546 | |
547 | if (err_data.ue_count) |
548 | dev_info(adev->dev, "%ld uncorrectable hardware " |
549 | "errors detected in %s block\n" , |
550 | obj->err_data.ue_count, |
551 | get_ras_block_str(adev->nbio.ras_if)); |
552 | } |
553 | |
554 | dev_info(adev->dev, "RAS controller interrupt triggered " |
555 | "by NBIF error\n" ); |
556 | } |
557 | |
558 | amdgpu_ras_error_data_fini(err_data: &err_data); |
559 | } |
560 | |
561 | static void nbio_v7_9_handle_ras_err_event_athub_intr_no_bifring(struct amdgpu_device *adev) |
562 | { |
563 | uint32_t bif_doorbell_intr_cntl; |
564 | |
565 | bif_doorbell_intr_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_DOORBELL_INT_CNTL); |
566 | |
567 | if (REG_GET_FIELD(bif_doorbell_intr_cntl, |
568 | BIF_BX0_BIF_DOORBELL_INT_CNTL, RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS)) { |
569 | /* driver has to clear the interrupt status when bif ring is disabled */ |
570 | bif_doorbell_intr_cntl = REG_SET_FIELD(bif_doorbell_intr_cntl, |
571 | BIF_BX0_BIF_DOORBELL_INT_CNTL, |
572 | RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR, 1); |
573 | |
574 | WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_DOORBELL_INT_CNTL, bif_doorbell_intr_cntl); |
575 | |
576 | amdgpu_ras_global_ras_isr(adev); |
577 | } |
578 | } |
579 | |
580 | static int nbio_v7_9_set_ras_controller_irq_state(struct amdgpu_device *adev, |
581 | struct amdgpu_irq_src *src, |
582 | unsigned type, |
583 | enum amdgpu_interrupt_state state) |
584 | { |
585 | /* Dummy function, there is no initialization operation in driver */ |
586 | |
587 | return 0; |
588 | } |
589 | |
590 | static int nbio_v7_9_process_ras_controller_irq(struct amdgpu_device *adev, |
591 | struct amdgpu_irq_src *source, |
592 | struct amdgpu_iv_entry *entry) |
593 | { |
594 | /* By design, the ih cookie for ras_controller_irq should be written |
595 | * to BIFring instead of general iv ring. However, due to known bif ring |
596 | * hw bug, it has to be disabled. There is no chance the process function |
597 | * will be involked. Just left it as a dummy one. |
598 | */ |
599 | return 0; |
600 | } |
601 | |
602 | static int nbio_v7_9_set_ras_err_event_athub_irq_state(struct amdgpu_device *adev, |
603 | struct amdgpu_irq_src *src, |
604 | unsigned type, |
605 | enum amdgpu_interrupt_state state) |
606 | { |
607 | /* Dummy function, there is no initialization operation in driver */ |
608 | |
609 | return 0; |
610 | } |
611 | |
612 | static int nbio_v7_9_process_err_event_athub_irq(struct amdgpu_device *adev, |
613 | struct amdgpu_irq_src *source, |
614 | struct amdgpu_iv_entry *entry) |
615 | { |
616 | /* By design, the ih cookie for err_event_athub_irq should be written |
617 | * to BIFring instead of general iv ring. However, due to known bif ring |
618 | * hw bug, it has to be disabled. There is no chance the process function |
619 | * will be involked. Just left it as a dummy one. |
620 | */ |
621 | return 0; |
622 | } |
623 | |
624 | static const struct amdgpu_irq_src_funcs nbio_v7_9_ras_controller_irq_funcs = { |
625 | .set = nbio_v7_9_set_ras_controller_irq_state, |
626 | .process = nbio_v7_9_process_ras_controller_irq, |
627 | }; |
628 | |
629 | static const struct amdgpu_irq_src_funcs nbio_v7_9_ras_err_event_athub_irq_funcs = { |
630 | .set = nbio_v7_9_set_ras_err_event_athub_irq_state, |
631 | .process = nbio_v7_9_process_err_event_athub_irq, |
632 | }; |
633 | |
634 | static int nbio_v7_9_init_ras_controller_interrupt (struct amdgpu_device *adev) |
635 | { |
636 | int r; |
637 | |
638 | /* init the irq funcs */ |
639 | adev->nbio.ras_controller_irq.funcs = |
640 | &nbio_v7_9_ras_controller_irq_funcs; |
641 | adev->nbio.ras_controller_irq.num_types = 1; |
642 | |
643 | /* register ras controller interrupt */ |
644 | r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_BIF, |
645 | NBIF_7_4__SRCID__RAS_CONTROLLER_INTERRUPT, |
646 | source: &adev->nbio.ras_controller_irq); |
647 | |
648 | return r; |
649 | } |
650 | |
651 | static int nbio_v7_9_init_ras_err_event_athub_interrupt (struct amdgpu_device *adev) |
652 | { |
653 | |
654 | int r; |
655 | |
656 | /* init the irq funcs */ |
657 | adev->nbio.ras_err_event_athub_irq.funcs = |
658 | &nbio_v7_9_ras_err_event_athub_irq_funcs; |
659 | adev->nbio.ras_err_event_athub_irq.num_types = 1; |
660 | |
661 | /* register ras err event athub interrupt */ |
662 | r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_BIF, |
663 | NBIF_7_4__SRCID__ERREVENT_ATHUB_INTERRUPT, |
664 | source: &adev->nbio.ras_err_event_athub_irq); |
665 | |
666 | return r; |
667 | } |
668 | |
669 | const struct amdgpu_ras_block_hw_ops nbio_v7_9_ras_hw_ops = { |
670 | .query_ras_error_count = nbio_v7_9_query_ras_error_count, |
671 | }; |
672 | |
673 | struct amdgpu_nbio_ras nbio_v7_9_ras = { |
674 | .ras_block = { |
675 | .ras_comm = { |
676 | .name = "pcie_bif" , |
677 | .block = AMDGPU_RAS_BLOCK__PCIE_BIF, |
678 | .type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, |
679 | }, |
680 | .hw_ops = &nbio_v7_9_ras_hw_ops, |
681 | .ras_late_init = amdgpu_nbio_ras_late_init, |
682 | }, |
683 | .handle_ras_controller_intr_no_bifring = nbio_v7_9_handle_ras_controller_intr_no_bifring, |
684 | .handle_ras_err_event_athub_intr_no_bifring = nbio_v7_9_handle_ras_err_event_athub_intr_no_bifring, |
685 | .init_ras_controller_interrupt = nbio_v7_9_init_ras_controller_interrupt, |
686 | .init_ras_err_event_athub_interrupt = nbio_v7_9_init_ras_err_event_athub_interrupt, |
687 | }; |
688 | |