1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | |
24 | #include <linux/firmware.h> |
25 | #include <linux/module.h> |
26 | #include <linux/pci.h> |
27 | |
28 | #include <drm/drm_cache.h> |
29 | #include "amdgpu.h" |
30 | #include "gmc_v8_0.h" |
31 | #include "amdgpu_ucode.h" |
32 | #include "amdgpu_amdkfd.h" |
33 | #include "amdgpu_gem.h" |
34 | |
35 | #include "gmc/gmc_8_1_d.h" |
36 | #include "gmc/gmc_8_1_sh_mask.h" |
37 | |
38 | #include "bif/bif_5_0_d.h" |
39 | #include "bif/bif_5_0_sh_mask.h" |
40 | |
41 | #include "oss/oss_3_0_d.h" |
42 | #include "oss/oss_3_0_sh_mask.h" |
43 | |
44 | #include "dce/dce_10_0_d.h" |
45 | #include "dce/dce_10_0_sh_mask.h" |
46 | |
47 | #include "vid.h" |
48 | #include "vi.h" |
49 | |
50 | #include "amdgpu_atombios.h" |
51 | |
52 | #include "ivsrcid/ivsrcid_vislands30.h" |
53 | |
54 | static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device *adev); |
55 | static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev); |
56 | static int gmc_v8_0_wait_for_idle(void *handle); |
57 | |
58 | MODULE_FIRMWARE("amdgpu/tonga_mc.bin" ); |
59 | MODULE_FIRMWARE("amdgpu/polaris11_mc.bin" ); |
60 | MODULE_FIRMWARE("amdgpu/polaris10_mc.bin" ); |
61 | MODULE_FIRMWARE("amdgpu/polaris12_mc.bin" ); |
62 | MODULE_FIRMWARE("amdgpu/polaris12_32_mc.bin" ); |
63 | MODULE_FIRMWARE("amdgpu/polaris11_k_mc.bin" ); |
64 | MODULE_FIRMWARE("amdgpu/polaris10_k_mc.bin" ); |
65 | MODULE_FIRMWARE("amdgpu/polaris12_k_mc.bin" ); |
66 | |
67 | static const u32 golden_settings_tonga_a11[] = { |
68 | mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000, |
69 | mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028, |
70 | mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991, |
71 | mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
72 | mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
73 | mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
74 | mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
75 | }; |
76 | |
77 | static const u32 tonga_mgcg_cgcg_init[] = { |
78 | mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 |
79 | }; |
80 | |
81 | static const u32 golden_settings_fiji_a10[] = { |
82 | mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
83 | mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
84 | mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
85 | mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
86 | }; |
87 | |
88 | static const u32 fiji_mgcg_cgcg_init[] = { |
89 | mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 |
90 | }; |
91 | |
92 | static const u32 golden_settings_polaris11_a11[] = { |
93 | mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
94 | mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
95 | mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
96 | mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff |
97 | }; |
98 | |
99 | static const u32 golden_settings_polaris10_a11[] = { |
100 | mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000, |
101 | mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
102 | mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
103 | mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff, |
104 | mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff |
105 | }; |
106 | |
107 | static const u32 cz_mgcg_cgcg_init[] = { |
108 | mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 |
109 | }; |
110 | |
111 | static const u32 stoney_mgcg_cgcg_init[] = { |
112 | mmATC_MISC_CG, 0xffffffff, 0x000c0200, |
113 | mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 |
114 | }; |
115 | |
116 | static const u32 golden_settings_stoney_common[] = { |
117 | mmMC_HUB_RDREQ_UVD, MC_HUB_RDREQ_UVD__PRESCALE_MASK, 0x00000004, |
118 | mmMC_RD_GRP_OTH, MC_RD_GRP_OTH__UVD_MASK, 0x00600000 |
119 | }; |
120 | |
121 | static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev) |
122 | { |
123 | switch (adev->asic_type) { |
124 | case CHIP_FIJI: |
125 | amdgpu_device_program_register_sequence(adev, |
126 | registers: fiji_mgcg_cgcg_init, |
127 | ARRAY_SIZE(fiji_mgcg_cgcg_init)); |
128 | amdgpu_device_program_register_sequence(adev, |
129 | registers: golden_settings_fiji_a10, |
130 | ARRAY_SIZE(golden_settings_fiji_a10)); |
131 | break; |
132 | case CHIP_TONGA: |
133 | amdgpu_device_program_register_sequence(adev, |
134 | registers: tonga_mgcg_cgcg_init, |
135 | ARRAY_SIZE(tonga_mgcg_cgcg_init)); |
136 | amdgpu_device_program_register_sequence(adev, |
137 | registers: golden_settings_tonga_a11, |
138 | ARRAY_SIZE(golden_settings_tonga_a11)); |
139 | break; |
140 | case CHIP_POLARIS11: |
141 | case CHIP_POLARIS12: |
142 | case CHIP_VEGAM: |
143 | amdgpu_device_program_register_sequence(adev, |
144 | registers: golden_settings_polaris11_a11, |
145 | ARRAY_SIZE(golden_settings_polaris11_a11)); |
146 | break; |
147 | case CHIP_POLARIS10: |
148 | amdgpu_device_program_register_sequence(adev, |
149 | registers: golden_settings_polaris10_a11, |
150 | ARRAY_SIZE(golden_settings_polaris10_a11)); |
151 | break; |
152 | case CHIP_CARRIZO: |
153 | amdgpu_device_program_register_sequence(adev, |
154 | registers: cz_mgcg_cgcg_init, |
155 | ARRAY_SIZE(cz_mgcg_cgcg_init)); |
156 | break; |
157 | case CHIP_STONEY: |
158 | amdgpu_device_program_register_sequence(adev, |
159 | registers: stoney_mgcg_cgcg_init, |
160 | ARRAY_SIZE(stoney_mgcg_cgcg_init)); |
161 | amdgpu_device_program_register_sequence(adev, |
162 | registers: golden_settings_stoney_common, |
163 | ARRAY_SIZE(golden_settings_stoney_common)); |
164 | break; |
165 | default: |
166 | break; |
167 | } |
168 | } |
169 | |
170 | static void gmc_v8_0_mc_stop(struct amdgpu_device *adev) |
171 | { |
172 | u32 blackout; |
173 | |
174 | gmc_v8_0_wait_for_idle(handle: adev); |
175 | |
176 | blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL); |
177 | if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) { |
178 | /* Block CPU access */ |
179 | WREG32(mmBIF_FB_EN, 0); |
180 | /* blackout the MC */ |
181 | blackout = REG_SET_FIELD(blackout, |
182 | MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 1); |
183 | WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout); |
184 | } |
185 | /* wait for the MC to settle */ |
186 | udelay(100); |
187 | } |
188 | |
189 | static void gmc_v8_0_mc_resume(struct amdgpu_device *adev) |
190 | { |
191 | u32 tmp; |
192 | |
193 | /* unblackout the MC */ |
194 | tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL); |
195 | tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0); |
196 | WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp); |
197 | /* allow CPU access */ |
198 | tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1); |
199 | tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1); |
200 | WREG32(mmBIF_FB_EN, tmp); |
201 | } |
202 | |
203 | /** |
204 | * gmc_v8_0_init_microcode - load ucode images from disk |
205 | * |
206 | * @adev: amdgpu_device pointer |
207 | * |
208 | * Use the firmware interface to load the ucode images into |
209 | * the driver (not loaded into hw). |
210 | * Returns 0 on success, error on failure. |
211 | */ |
212 | static int gmc_v8_0_init_microcode(struct amdgpu_device *adev) |
213 | { |
214 | const char *chip_name; |
215 | char fw_name[30]; |
216 | int err; |
217 | |
218 | DRM_DEBUG("\n" ); |
219 | |
220 | switch (adev->asic_type) { |
221 | case CHIP_TONGA: |
222 | chip_name = "tonga" ; |
223 | break; |
224 | case CHIP_POLARIS11: |
225 | if (ASICID_IS_P21(adev->pdev->device, adev->pdev->revision) || |
226 | ASICID_IS_P31(adev->pdev->device, adev->pdev->revision)) |
227 | chip_name = "polaris11_k" ; |
228 | else |
229 | chip_name = "polaris11" ; |
230 | break; |
231 | case CHIP_POLARIS10: |
232 | if (ASICID_IS_P30(adev->pdev->device, adev->pdev->revision)) |
233 | chip_name = "polaris10_k" ; |
234 | else |
235 | chip_name = "polaris10" ; |
236 | break; |
237 | case CHIP_POLARIS12: |
238 | if (ASICID_IS_P23(adev->pdev->device, adev->pdev->revision)) { |
239 | chip_name = "polaris12_k" ; |
240 | } else { |
241 | WREG32(mmMC_SEQ_IO_DEBUG_INDEX, ixMC_IO_DEBUG_UP_159); |
242 | /* Polaris12 32bit ASIC needs a special MC firmware */ |
243 | if (RREG32(mmMC_SEQ_IO_DEBUG_DATA) == 0x05b4dc40) |
244 | chip_name = "polaris12_32" ; |
245 | else |
246 | chip_name = "polaris12" ; |
247 | } |
248 | break; |
249 | case CHIP_FIJI: |
250 | case CHIP_CARRIZO: |
251 | case CHIP_STONEY: |
252 | case CHIP_VEGAM: |
253 | return 0; |
254 | default: |
255 | return -EINVAL; |
256 | } |
257 | |
258 | snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_mc.bin" , chip_name); |
259 | err = amdgpu_ucode_request(adev, fw: &adev->gmc.fw, fw_name); |
260 | if (err) { |
261 | pr_err("mc: Failed to load firmware \"%s\"\n" , fw_name); |
262 | amdgpu_ucode_release(fw: &adev->gmc.fw); |
263 | } |
264 | return err; |
265 | } |
266 | |
267 | /** |
268 | * gmc_v8_0_tonga_mc_load_microcode - load tonga MC ucode into the hw |
269 | * |
270 | * @adev: amdgpu_device pointer |
271 | * |
272 | * Load the GDDR MC ucode into the hw (VI). |
273 | * Returns 0 on success, error on failure. |
274 | */ |
275 | static int gmc_v8_0_tonga_mc_load_microcode(struct amdgpu_device *adev) |
276 | { |
277 | const struct mc_firmware_header_v1_0 *hdr; |
278 | const __le32 *fw_data = NULL; |
279 | const __le32 *io_mc_regs = NULL; |
280 | u32 running; |
281 | int i, ucode_size, regs_size; |
282 | |
283 | /* Skip MC ucode loading on SR-IOV capable boards. |
284 | * vbios does this for us in asic_init in that case. |
285 | * Skip MC ucode loading on VF, because hypervisor will do that |
286 | * for this adaptor. |
287 | */ |
288 | if (amdgpu_sriov_bios(adev)) |
289 | return 0; |
290 | |
291 | if (!adev->gmc.fw) |
292 | return -EINVAL; |
293 | |
294 | hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data; |
295 | amdgpu_ucode_print_mc_hdr(hdr: &hdr->header); |
296 | |
297 | adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version); |
298 | regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); |
299 | io_mc_regs = (const __le32 *) |
300 | (adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); |
301 | ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; |
302 | fw_data = (const __le32 *) |
303 | (adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); |
304 | |
305 | running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN); |
306 | |
307 | if (running == 0) { |
308 | /* reset the engine and set to writable */ |
309 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008); |
310 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010); |
311 | |
312 | /* load mc io regs */ |
313 | for (i = 0; i < regs_size; i++) { |
314 | WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++)); |
315 | WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++)); |
316 | } |
317 | /* load the MC ucode */ |
318 | for (i = 0; i < ucode_size; i++) |
319 | WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++)); |
320 | |
321 | /* put the engine back into the active state */ |
322 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008); |
323 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004); |
324 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001); |
325 | |
326 | /* wait for training to complete */ |
327 | for (i = 0; i < adev->usec_timeout; i++) { |
328 | if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL), |
329 | MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0)) |
330 | break; |
331 | udelay(1); |
332 | } |
333 | for (i = 0; i < adev->usec_timeout; i++) { |
334 | if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL), |
335 | MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1)) |
336 | break; |
337 | udelay(1); |
338 | } |
339 | } |
340 | |
341 | return 0; |
342 | } |
343 | |
344 | static int gmc_v8_0_polaris_mc_load_microcode(struct amdgpu_device *adev) |
345 | { |
346 | const struct mc_firmware_header_v1_0 *hdr; |
347 | const __le32 *fw_data = NULL; |
348 | const __le32 *io_mc_regs = NULL; |
349 | u32 data; |
350 | int i, ucode_size, regs_size; |
351 | |
352 | /* Skip MC ucode loading on SR-IOV capable boards. |
353 | * vbios does this for us in asic_init in that case. |
354 | * Skip MC ucode loading on VF, because hypervisor will do that |
355 | * for this adaptor. |
356 | */ |
357 | if (amdgpu_sriov_bios(adev)) |
358 | return 0; |
359 | |
360 | if (!adev->gmc.fw) |
361 | return -EINVAL; |
362 | |
363 | hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data; |
364 | amdgpu_ucode_print_mc_hdr(hdr: &hdr->header); |
365 | |
366 | adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version); |
367 | regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2); |
368 | io_mc_regs = (const __le32 *) |
369 | (adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); |
370 | ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; |
371 | fw_data = (const __le32 *) |
372 | (adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); |
373 | |
374 | data = RREG32(mmMC_SEQ_MISC0); |
375 | data &= ~(0x40); |
376 | WREG32(mmMC_SEQ_MISC0, data); |
377 | |
378 | /* load mc io regs */ |
379 | for (i = 0; i < regs_size; i++) { |
380 | WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++)); |
381 | WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++)); |
382 | } |
383 | |
384 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008); |
385 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010); |
386 | |
387 | /* load the MC ucode */ |
388 | for (i = 0; i < ucode_size; i++) |
389 | WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++)); |
390 | |
391 | /* put the engine back into the active state */ |
392 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008); |
393 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004); |
394 | WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001); |
395 | |
396 | /* wait for training to complete */ |
397 | for (i = 0; i < adev->usec_timeout; i++) { |
398 | data = RREG32(mmMC_SEQ_MISC0); |
399 | if (data & 0x80) |
400 | break; |
401 | udelay(1); |
402 | } |
403 | |
404 | return 0; |
405 | } |
406 | |
407 | static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev, |
408 | struct amdgpu_gmc *mc) |
409 | { |
410 | u64 base = 0; |
411 | |
412 | if (!amdgpu_sriov_vf(adev)) |
413 | base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF; |
414 | base <<= 24; |
415 | |
416 | amdgpu_gmc_vram_location(adev, mc, base); |
417 | amdgpu_gmc_gart_location(adev, mc, gart_placement: AMDGPU_GART_PLACEMENT_BEST_FIT); |
418 | } |
419 | |
420 | /** |
421 | * gmc_v8_0_mc_program - program the GPU memory controller |
422 | * |
423 | * @adev: amdgpu_device pointer |
424 | * |
425 | * Set the location of vram, gart, and AGP in the GPU's |
426 | * physical address space (VI). |
427 | */ |
428 | static void gmc_v8_0_mc_program(struct amdgpu_device *adev) |
429 | { |
430 | u32 tmp; |
431 | int i, j; |
432 | |
433 | /* Initialize HDP */ |
434 | for (i = 0, j = 0; i < 32; i++, j += 0x6) { |
435 | WREG32((0xb05 + j), 0x00000000); |
436 | WREG32((0xb06 + j), 0x00000000); |
437 | WREG32((0xb07 + j), 0x00000000); |
438 | WREG32((0xb08 + j), 0x00000000); |
439 | WREG32((0xb09 + j), 0x00000000); |
440 | } |
441 | WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0); |
442 | |
443 | if (gmc_v8_0_wait_for_idle(handle: (void *)adev)) |
444 | dev_warn(adev->dev, "Wait for MC idle timedout !\n" ); |
445 | |
446 | if (adev->mode_info.num_crtc) { |
447 | /* Lockout access through VGA aperture*/ |
448 | tmp = RREG32(mmVGA_HDP_CONTROL); |
449 | tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); |
450 | WREG32(mmVGA_HDP_CONTROL, tmp); |
451 | |
452 | /* disable VGA render */ |
453 | tmp = RREG32(mmVGA_RENDER_CONTROL); |
454 | tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); |
455 | WREG32(mmVGA_RENDER_CONTROL, tmp); |
456 | } |
457 | /* Update configuration */ |
458 | WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR, |
459 | adev->gmc.vram_start >> 12); |
460 | WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR, |
461 | adev->gmc.vram_end >> 12); |
462 | WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, |
463 | adev->mem_scratch.gpu_addr >> 12); |
464 | |
465 | if (amdgpu_sriov_vf(adev)) { |
466 | tmp = ((adev->gmc.vram_end >> 24) & 0xFFFF) << 16; |
467 | tmp |= ((adev->gmc.vram_start >> 24) & 0xFFFF); |
468 | WREG32(mmMC_VM_FB_LOCATION, tmp); |
469 | /* XXX double check these! */ |
470 | WREG32(mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); |
471 | WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); |
472 | WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF); |
473 | } |
474 | |
475 | WREG32(mmMC_VM_AGP_BASE, 0); |
476 | WREG32(mmMC_VM_AGP_TOP, adev->gmc.agp_end >> 22); |
477 | WREG32(mmMC_VM_AGP_BOT, adev->gmc.agp_start >> 22); |
478 | if (gmc_v8_0_wait_for_idle(handle: (void *)adev)) |
479 | dev_warn(adev->dev, "Wait for MC idle timedout !\n" ); |
480 | |
481 | WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK); |
482 | |
483 | tmp = RREG32(mmHDP_MISC_CNTL); |
484 | tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0); |
485 | WREG32(mmHDP_MISC_CNTL, tmp); |
486 | |
487 | tmp = RREG32(mmHDP_HOST_PATH_CNTL); |
488 | WREG32(mmHDP_HOST_PATH_CNTL, tmp); |
489 | } |
490 | |
491 | /** |
492 | * gmc_v8_0_mc_init - initialize the memory controller driver params |
493 | * |
494 | * @adev: amdgpu_device pointer |
495 | * |
496 | * Look up the amount of vram, vram width, and decide how to place |
497 | * vram and gart within the GPU's physical address space (VI). |
498 | * Returns 0 for success. |
499 | */ |
500 | static int gmc_v8_0_mc_init(struct amdgpu_device *adev) |
501 | { |
502 | int r; |
503 | u32 tmp; |
504 | |
505 | adev->gmc.vram_width = amdgpu_atombios_get_vram_width(adev); |
506 | if (!adev->gmc.vram_width) { |
507 | int chansize, numchan; |
508 | |
509 | /* Get VRAM informations */ |
510 | tmp = RREG32(mmMC_ARB_RAMCFG); |
511 | if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) |
512 | chansize = 64; |
513 | else |
514 | chansize = 32; |
515 | |
516 | tmp = RREG32(mmMC_SHARED_CHMAP); |
517 | switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) { |
518 | case 0: |
519 | default: |
520 | numchan = 1; |
521 | break; |
522 | case 1: |
523 | numchan = 2; |
524 | break; |
525 | case 2: |
526 | numchan = 4; |
527 | break; |
528 | case 3: |
529 | numchan = 8; |
530 | break; |
531 | case 4: |
532 | numchan = 3; |
533 | break; |
534 | case 5: |
535 | numchan = 6; |
536 | break; |
537 | case 6: |
538 | numchan = 10; |
539 | break; |
540 | case 7: |
541 | numchan = 12; |
542 | break; |
543 | case 8: |
544 | numchan = 16; |
545 | break; |
546 | } |
547 | adev->gmc.vram_width = numchan * chansize; |
548 | } |
549 | /* size in MB on si */ |
550 | tmp = RREG32(mmCONFIG_MEMSIZE); |
551 | /* some boards may have garbage in the upper 16 bits */ |
552 | if (tmp & 0xffff0000) { |
553 | DRM_INFO("Probable bad vram size: 0x%08x\n" , tmp); |
554 | if (tmp & 0xffff) |
555 | tmp &= 0xffff; |
556 | } |
557 | adev->gmc.mc_vram_size = tmp * 1024ULL * 1024ULL; |
558 | adev->gmc.real_vram_size = adev->gmc.mc_vram_size; |
559 | |
560 | if (!(adev->flags & AMD_IS_APU)) { |
561 | r = amdgpu_device_resize_fb_bar(adev); |
562 | if (r) |
563 | return r; |
564 | } |
565 | adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); |
566 | adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); |
567 | |
568 | #ifdef CONFIG_X86_64 |
569 | if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) { |
570 | adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22; |
571 | adev->gmc.aper_size = adev->gmc.real_vram_size; |
572 | } |
573 | #endif |
574 | |
575 | adev->gmc.visible_vram_size = adev->gmc.aper_size; |
576 | |
577 | /* set the gart size */ |
578 | if (amdgpu_gart_size == -1) { |
579 | switch (adev->asic_type) { |
580 | case CHIP_POLARIS10: /* all engines support GPUVM */ |
581 | case CHIP_POLARIS11: /* all engines support GPUVM */ |
582 | case CHIP_POLARIS12: /* all engines support GPUVM */ |
583 | case CHIP_VEGAM: /* all engines support GPUVM */ |
584 | default: |
585 | adev->gmc.gart_size = 256ULL << 20; |
586 | break; |
587 | case CHIP_TONGA: /* UVD, VCE do not support GPUVM */ |
588 | case CHIP_FIJI: /* UVD, VCE do not support GPUVM */ |
589 | case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */ |
590 | case CHIP_STONEY: /* UVD does not support GPUVM, DCE SG support */ |
591 | adev->gmc.gart_size = 1024ULL << 20; |
592 | break; |
593 | } |
594 | } else { |
595 | adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; |
596 | } |
597 | |
598 | adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; |
599 | gmc_v8_0_vram_gtt_location(adev, mc: &adev->gmc); |
600 | |
601 | return 0; |
602 | } |
603 | |
604 | /** |
605 | * gmc_v8_0_flush_gpu_tlb_pasid - tlb flush via pasid |
606 | * |
607 | * @adev: amdgpu_device pointer |
608 | * @pasid: pasid to be flush |
609 | * @flush_type: type of flush |
610 | * @all_hub: flush all hubs |
611 | * @inst: is used to select which instance of KIQ to use for the invalidation |
612 | * |
613 | * Flush the TLB for the requested pasid. |
614 | */ |
615 | static void gmc_v8_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, |
616 | uint16_t pasid, uint32_t flush_type, |
617 | bool all_hub, uint32_t inst) |
618 | { |
619 | u32 mask = 0x0; |
620 | int vmid; |
621 | |
622 | for (vmid = 1; vmid < 16; vmid++) { |
623 | u32 tmp = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); |
624 | |
625 | if ((tmp & ATC_VMID0_PASID_MAPPING__VALID_MASK) && |
626 | (tmp & ATC_VMID0_PASID_MAPPING__PASID_MASK) == pasid) |
627 | mask |= 1 << vmid; |
628 | } |
629 | |
630 | WREG32(mmVM_INVALIDATE_REQUEST, mask); |
631 | RREG32(mmVM_INVALIDATE_RESPONSE); |
632 | } |
633 | |
634 | /* |
635 | * GART |
636 | * VMID 0 is the physical GPU addresses as used by the kernel. |
637 | * VMIDs 1-15 are used for userspace clients and are handled |
638 | * by the amdgpu vm/hsa code. |
639 | */ |
640 | |
641 | /** |
642 | * gmc_v8_0_flush_gpu_tlb - gart tlb flush callback |
643 | * |
644 | * @adev: amdgpu_device pointer |
645 | * @vmid: vm instance to flush |
646 | * @vmhub: which hub to flush |
647 | * @flush_type: type of flush |
648 | * |
649 | * Flush the TLB for the requested page table (VI). |
650 | */ |
651 | static void gmc_v8_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, |
652 | uint32_t vmhub, uint32_t flush_type) |
653 | { |
654 | /* bits 0-15 are the VM contexts0-15 */ |
655 | WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); |
656 | } |
657 | |
658 | static uint64_t gmc_v8_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, |
659 | unsigned int vmid, uint64_t pd_addr) |
660 | { |
661 | uint32_t reg; |
662 | |
663 | if (vmid < 8) |
664 | reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid; |
665 | else |
666 | reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8; |
667 | amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12); |
668 | |
669 | /* bits 0-15 are the VM contexts0-15 */ |
670 | amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid); |
671 | |
672 | return pd_addr; |
673 | } |
674 | |
675 | static void gmc_v8_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid, |
676 | unsigned int pasid) |
677 | { |
678 | amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid); |
679 | } |
680 | |
681 | /* |
682 | * PTE format on VI: |
683 | * 63:40 reserved |
684 | * 39:12 4k physical page base address |
685 | * 11:7 fragment |
686 | * 6 write |
687 | * 5 read |
688 | * 4 exe |
689 | * 3 reserved |
690 | * 2 snooped |
691 | * 1 system |
692 | * 0 valid |
693 | * |
694 | * PDE format on VI: |
695 | * 63:59 block fragment size |
696 | * 58:40 reserved |
697 | * 39:1 physical base address of PTE |
698 | * bits 5:1 must be 0. |
699 | * 0 valid |
700 | */ |
701 | |
702 | static void gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, int level, |
703 | uint64_t *addr, uint64_t *flags) |
704 | { |
705 | BUG_ON(*addr & 0xFFFFFF0000000FFFULL); |
706 | } |
707 | |
708 | static void gmc_v8_0_get_vm_pte(struct amdgpu_device *adev, |
709 | struct amdgpu_bo_va_mapping *mapping, |
710 | uint64_t *flags) |
711 | { |
712 | *flags &= ~AMDGPU_PTE_EXECUTABLE; |
713 | *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; |
714 | *flags &= ~AMDGPU_PTE_PRT; |
715 | } |
716 | |
717 | /** |
718 | * gmc_v8_0_set_fault_enable_default - update VM fault handling |
719 | * |
720 | * @adev: amdgpu_device pointer |
721 | * @value: true redirects VM faults to the default page |
722 | */ |
723 | static void gmc_v8_0_set_fault_enable_default(struct amdgpu_device *adev, |
724 | bool value) |
725 | { |
726 | u32 tmp; |
727 | |
728 | tmp = RREG32(mmVM_CONTEXT1_CNTL); |
729 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
730 | RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
731 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
732 | DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
733 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
734 | PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
735 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
736 | VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
737 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
738 | READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
739 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
740 | WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
741 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
742 | EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
743 | WREG32(mmVM_CONTEXT1_CNTL, tmp); |
744 | } |
745 | |
746 | /** |
747 | * gmc_v8_0_set_prt() - set PRT VM fault |
748 | * |
749 | * @adev: amdgpu_device pointer |
750 | * @enable: enable/disable VM fault handling for PRT |
751 | */ |
752 | static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable) |
753 | { |
754 | u32 tmp; |
755 | |
756 | if (enable && !adev->gmc.prt_warning) { |
757 | dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n" ); |
758 | adev->gmc.prt_warning = true; |
759 | } |
760 | |
761 | tmp = RREG32(mmVM_PRT_CNTL); |
762 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, |
763 | CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable); |
764 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, |
765 | CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable); |
766 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, |
767 | TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable); |
768 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, |
769 | TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable); |
770 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, |
771 | L2_CACHE_STORE_INVALID_ENTRIES, enable); |
772 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, |
773 | L1_TLB_STORE_INVALID_ENTRIES, enable); |
774 | tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL, |
775 | MASK_PDE0_FAULT, enable); |
776 | WREG32(mmVM_PRT_CNTL, tmp); |
777 | |
778 | if (enable) { |
779 | uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT; |
780 | uint32_t high = adev->vm_manager.max_pfn - |
781 | (AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT); |
782 | |
783 | WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low); |
784 | WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low); |
785 | WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low); |
786 | WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low); |
787 | WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high); |
788 | WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high); |
789 | WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high); |
790 | WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high); |
791 | } else { |
792 | WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff); |
793 | WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff); |
794 | WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff); |
795 | WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff); |
796 | WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0); |
797 | WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0); |
798 | WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0); |
799 | WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0); |
800 | } |
801 | } |
802 | |
803 | /** |
804 | * gmc_v8_0_gart_enable - gart enable |
805 | * |
806 | * @adev: amdgpu_device pointer |
807 | * |
808 | * This sets up the TLBs, programs the page tables for VMID0, |
809 | * sets up the hw for VMIDs 1-15 which are allocated on |
810 | * demand, and sets up the global locations for the LDS, GDS, |
811 | * and GPUVM for FSA64 clients (VI). |
812 | * Returns 0 for success, errors for failure. |
813 | */ |
814 | static int gmc_v8_0_gart_enable(struct amdgpu_device *adev) |
815 | { |
816 | uint64_t table_addr; |
817 | u32 tmp, field; |
818 | int i; |
819 | |
820 | if (adev->gart.bo == NULL) { |
821 | dev_err(adev->dev, "No VRAM object for PCIE GART.\n" ); |
822 | return -EINVAL; |
823 | } |
824 | amdgpu_gtt_mgr_recover(mgr: &adev->mman.gtt_mgr); |
825 | table_addr = amdgpu_bo_gpu_offset(bo: adev->gart.bo); |
826 | |
827 | /* Setup TLB control */ |
828 | tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL); |
829 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1); |
830 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1); |
831 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3); |
832 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1); |
833 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); |
834 | WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp); |
835 | /* Setup L2 cache */ |
836 | tmp = RREG32(mmVM_L2_CNTL); |
837 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1); |
838 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1); |
839 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1); |
840 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1); |
841 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7); |
842 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1); |
843 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1); |
844 | WREG32(mmVM_L2_CNTL, tmp); |
845 | tmp = RREG32(mmVM_L2_CNTL2); |
846 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1); |
847 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); |
848 | WREG32(mmVM_L2_CNTL2, tmp); |
849 | |
850 | field = adev->vm_manager.fragment_size; |
851 | tmp = RREG32(mmVM_L2_CNTL3); |
852 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1); |
853 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field); |
854 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field); |
855 | WREG32(mmVM_L2_CNTL3, tmp); |
856 | /* XXX: set to enable PTE/PDE in system memory */ |
857 | tmp = RREG32(mmVM_L2_CNTL4); |
858 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_PHYSICAL, 0); |
859 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SHARED, 0); |
860 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SNOOP, 0); |
861 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_PHYSICAL, 0); |
862 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SHARED, 0); |
863 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SNOOP, 0); |
864 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_PHYSICAL, 0); |
865 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SHARED, 0); |
866 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SNOOP, 0); |
867 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_PHYSICAL, 0); |
868 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SHARED, 0); |
869 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SNOOP, 0); |
870 | WREG32(mmVM_L2_CNTL4, tmp); |
871 | /* setup context0 */ |
872 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12); |
873 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12); |
874 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, table_addr >> 12); |
875 | WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, |
876 | (u32)(adev->dummy_page_addr >> 12)); |
877 | WREG32(mmVM_CONTEXT0_CNTL2, 0); |
878 | tmp = RREG32(mmVM_CONTEXT0_CNTL); |
879 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); |
880 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0); |
881 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
882 | WREG32(mmVM_CONTEXT0_CNTL, tmp); |
883 | |
884 | WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR, 0); |
885 | WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR, 0); |
886 | WREG32(mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET, 0); |
887 | |
888 | /* empty context1-15 */ |
889 | /* FIXME start with 4G, once using 2 level pt switch to full |
890 | * vm size space |
891 | */ |
892 | /* set vm size, must be a multiple of 4 */ |
893 | WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); |
894 | WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1); |
895 | for (i = 1; i < AMDGPU_NUM_VMID; i++) { |
896 | if (i < 8) |
897 | WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i, |
898 | table_addr >> 12); |
899 | else |
900 | WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8, |
901 | table_addr >> 12); |
902 | } |
903 | |
904 | /* enable context1-15 */ |
905 | WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, |
906 | (u32)(adev->dummy_page_addr >> 12)); |
907 | WREG32(mmVM_CONTEXT1_CNTL2, 4); |
908 | tmp = RREG32(mmVM_CONTEXT1_CNTL); |
909 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); |
910 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1); |
911 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
912 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
913 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
914 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
915 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
916 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
917 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
918 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE, |
919 | adev->vm_manager.block_size - 9); |
920 | WREG32(mmVM_CONTEXT1_CNTL, tmp); |
921 | if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) |
922 | gmc_v8_0_set_fault_enable_default(adev, value: false); |
923 | else |
924 | gmc_v8_0_set_fault_enable_default(adev, value: true); |
925 | |
926 | gmc_v8_0_flush_gpu_tlb(adev, vmid: 0, vmhub: 0, flush_type: 0); |
927 | DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n" , |
928 | (unsigned int)(adev->gmc.gart_size >> 20), |
929 | (unsigned long long)table_addr); |
930 | return 0; |
931 | } |
932 | |
933 | static int gmc_v8_0_gart_init(struct amdgpu_device *adev) |
934 | { |
935 | int r; |
936 | |
937 | if (adev->gart.bo) { |
938 | WARN(1, "R600 PCIE GART already initialized\n" ); |
939 | return 0; |
940 | } |
941 | /* Initialize common gart structure */ |
942 | r = amdgpu_gart_init(adev); |
943 | if (r) |
944 | return r; |
945 | adev->gart.table_size = adev->gart.num_gpu_pages * 8; |
946 | adev->gart.gart_pte_flags = AMDGPU_PTE_EXECUTABLE; |
947 | return amdgpu_gart_table_vram_alloc(adev); |
948 | } |
949 | |
950 | /** |
951 | * gmc_v8_0_gart_disable - gart disable |
952 | * |
953 | * @adev: amdgpu_device pointer |
954 | * |
955 | * This disables all VM page table (VI). |
956 | */ |
957 | static void gmc_v8_0_gart_disable(struct amdgpu_device *adev) |
958 | { |
959 | u32 tmp; |
960 | |
961 | /* Disable all tables */ |
962 | WREG32(mmVM_CONTEXT0_CNTL, 0); |
963 | WREG32(mmVM_CONTEXT1_CNTL, 0); |
964 | /* Setup TLB control */ |
965 | tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL); |
966 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); |
967 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0); |
968 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0); |
969 | WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp); |
970 | /* Setup L2 cache */ |
971 | tmp = RREG32(mmVM_L2_CNTL); |
972 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0); |
973 | WREG32(mmVM_L2_CNTL, tmp); |
974 | WREG32(mmVM_L2_CNTL2, 0); |
975 | } |
976 | |
977 | /** |
978 | * gmc_v8_0_vm_decode_fault - print human readable fault info |
979 | * |
980 | * @adev: amdgpu_device pointer |
981 | * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value |
982 | * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value |
983 | * @mc_client: VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT register value |
984 | * @pasid: debug logging only - no functional use |
985 | * |
986 | * Print human readable fault information (VI). |
987 | */ |
988 | static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev, u32 status, |
989 | u32 addr, u32 mc_client, unsigned int pasid) |
990 | { |
991 | u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID); |
992 | u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, |
993 | PROTECTIONS); |
994 | char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff, |
995 | (mc_client >> 8) & 0xff, mc_client & 0xff, 0 }; |
996 | u32 mc_id; |
997 | |
998 | mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, |
999 | MEMORY_CLIENT_ID); |
1000 | |
1001 | dev_err(adev->dev, "VM fault (0x%02x, vmid %d, pasid %d) at page %u, %s from '%s' (0x%08x) (%d)\n" , |
1002 | protections, vmid, pasid, addr, |
1003 | REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, |
1004 | MEMORY_CLIENT_RW) ? |
1005 | "write" : "read" , block, mc_client, mc_id); |
1006 | } |
1007 | |
1008 | static int gmc_v8_0_convert_vram_type(int mc_seq_vram_type) |
1009 | { |
1010 | switch (mc_seq_vram_type) { |
1011 | case MC_SEQ_MISC0__MT__GDDR1: |
1012 | return AMDGPU_VRAM_TYPE_GDDR1; |
1013 | case MC_SEQ_MISC0__MT__DDR2: |
1014 | return AMDGPU_VRAM_TYPE_DDR2; |
1015 | case MC_SEQ_MISC0__MT__GDDR3: |
1016 | return AMDGPU_VRAM_TYPE_GDDR3; |
1017 | case MC_SEQ_MISC0__MT__GDDR4: |
1018 | return AMDGPU_VRAM_TYPE_GDDR4; |
1019 | case MC_SEQ_MISC0__MT__GDDR5: |
1020 | return AMDGPU_VRAM_TYPE_GDDR5; |
1021 | case MC_SEQ_MISC0__MT__HBM: |
1022 | return AMDGPU_VRAM_TYPE_HBM; |
1023 | case MC_SEQ_MISC0__MT__DDR3: |
1024 | return AMDGPU_VRAM_TYPE_DDR3; |
1025 | default: |
1026 | return AMDGPU_VRAM_TYPE_UNKNOWN; |
1027 | } |
1028 | } |
1029 | |
1030 | static int gmc_v8_0_early_init(void *handle) |
1031 | { |
1032 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1033 | |
1034 | gmc_v8_0_set_gmc_funcs(adev); |
1035 | gmc_v8_0_set_irq_funcs(adev); |
1036 | |
1037 | adev->gmc.shared_aperture_start = 0x2000000000000000ULL; |
1038 | adev->gmc.shared_aperture_end = |
1039 | adev->gmc.shared_aperture_start + (4ULL << 30) - 1; |
1040 | adev->gmc.private_aperture_start = |
1041 | adev->gmc.shared_aperture_end + 1; |
1042 | adev->gmc.private_aperture_end = |
1043 | adev->gmc.private_aperture_start + (4ULL << 30) - 1; |
1044 | adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF; |
1045 | |
1046 | return 0; |
1047 | } |
1048 | |
1049 | static int gmc_v8_0_late_init(void *handle) |
1050 | { |
1051 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1052 | |
1053 | if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS) |
1054 | return amdgpu_irq_get(adev, src: &adev->gmc.vm_fault, type: 0); |
1055 | else |
1056 | return 0; |
1057 | } |
1058 | |
1059 | static unsigned int gmc_v8_0_get_vbios_fb_size(struct amdgpu_device *adev) |
1060 | { |
1061 | u32 d1vga_control = RREG32(mmD1VGA_CONTROL); |
1062 | unsigned int size; |
1063 | |
1064 | if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { |
1065 | size = AMDGPU_VBIOS_VGA_ALLOCATION; |
1066 | } else { |
1067 | u32 viewport = RREG32(mmVIEWPORT_SIZE); |
1068 | |
1069 | size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) * |
1070 | REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) * |
1071 | 4); |
1072 | } |
1073 | |
1074 | return size; |
1075 | } |
1076 | |
1077 | #define mmMC_SEQ_MISC0_FIJI 0xA71 |
1078 | |
1079 | static int gmc_v8_0_sw_init(void *handle) |
1080 | { |
1081 | int r; |
1082 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1083 | |
1084 | set_bit(AMDGPU_GFXHUB(0), addr: adev->vmhubs_mask); |
1085 | |
1086 | if (adev->flags & AMD_IS_APU) { |
1087 | adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; |
1088 | } else { |
1089 | u32 tmp; |
1090 | |
1091 | if ((adev->asic_type == CHIP_FIJI) || |
1092 | (adev->asic_type == CHIP_VEGAM)) |
1093 | tmp = RREG32(mmMC_SEQ_MISC0_FIJI); |
1094 | else |
1095 | tmp = RREG32(mmMC_SEQ_MISC0); |
1096 | tmp &= MC_SEQ_MISC0__MT__MASK; |
1097 | adev->gmc.vram_type = gmc_v8_0_convert_vram_type(mc_seq_vram_type: tmp); |
1098 | } |
1099 | |
1100 | r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, source: &adev->gmc.vm_fault); |
1101 | if (r) |
1102 | return r; |
1103 | |
1104 | r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, source: &adev->gmc.vm_fault); |
1105 | if (r) |
1106 | return r; |
1107 | |
1108 | /* Adjust VM size here. |
1109 | * Currently set to 4GB ((1 << 20) 4k pages). |
1110 | * Max GPUVM size for cayman and SI is 40 bits. |
1111 | */ |
1112 | amdgpu_vm_adjust_size(adev, min_vm_size: 64, fragment_size_default: 9, max_level: 1, max_bits: 40); |
1113 | |
1114 | /* Set the internal MC address mask |
1115 | * This is the max address of the GPU's |
1116 | * internal address space. |
1117 | */ |
1118 | adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */ |
1119 | |
1120 | r = dma_set_mask_and_coherent(dev: adev->dev, DMA_BIT_MASK(40)); |
1121 | if (r) { |
1122 | pr_warn("No suitable DMA available\n" ); |
1123 | return r; |
1124 | } |
1125 | adev->need_swiotlb = drm_need_swiotlb(dma_bits: 40); |
1126 | |
1127 | r = gmc_v8_0_init_microcode(adev); |
1128 | if (r) { |
1129 | DRM_ERROR("Failed to load mc firmware!\n" ); |
1130 | return r; |
1131 | } |
1132 | |
1133 | r = gmc_v8_0_mc_init(adev); |
1134 | if (r) |
1135 | return r; |
1136 | |
1137 | amdgpu_gmc_get_vbios_allocations(adev); |
1138 | |
1139 | /* Memory manager */ |
1140 | r = amdgpu_bo_init(adev); |
1141 | if (r) |
1142 | return r; |
1143 | |
1144 | r = gmc_v8_0_gart_init(adev); |
1145 | if (r) |
1146 | return r; |
1147 | |
1148 | /* |
1149 | * number of VMs |
1150 | * VMID 0 is reserved for System |
1151 | * amdgpu graphics/compute will use VMIDs 1-7 |
1152 | * amdkfd will use VMIDs 8-15 |
1153 | */ |
1154 | adev->vm_manager.first_kfd_vmid = 8; |
1155 | amdgpu_vm_manager_init(adev); |
1156 | |
1157 | /* base offset of vram pages */ |
1158 | if (adev->flags & AMD_IS_APU) { |
1159 | u64 tmp = RREG32(mmMC_VM_FB_OFFSET); |
1160 | |
1161 | tmp <<= 22; |
1162 | adev->vm_manager.vram_base_offset = tmp; |
1163 | } else { |
1164 | adev->vm_manager.vram_base_offset = 0; |
1165 | } |
1166 | |
1167 | adev->gmc.vm_fault_info = kmalloc(size: sizeof(struct kfd_vm_fault_info), |
1168 | GFP_KERNEL); |
1169 | if (!adev->gmc.vm_fault_info) |
1170 | return -ENOMEM; |
1171 | atomic_set(v: &adev->gmc.vm_fault_info_updated, i: 0); |
1172 | |
1173 | return 0; |
1174 | } |
1175 | |
1176 | static int gmc_v8_0_sw_fini(void *handle) |
1177 | { |
1178 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1179 | |
1180 | amdgpu_gem_force_release(adev); |
1181 | amdgpu_vm_manager_fini(adev); |
1182 | kfree(objp: adev->gmc.vm_fault_info); |
1183 | amdgpu_gart_table_vram_free(adev); |
1184 | amdgpu_bo_fini(adev); |
1185 | amdgpu_ucode_release(fw: &adev->gmc.fw); |
1186 | |
1187 | return 0; |
1188 | } |
1189 | |
1190 | static int gmc_v8_0_hw_init(void *handle) |
1191 | { |
1192 | int r; |
1193 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1194 | |
1195 | gmc_v8_0_init_golden_registers(adev); |
1196 | |
1197 | gmc_v8_0_mc_program(adev); |
1198 | |
1199 | if (adev->asic_type == CHIP_TONGA) { |
1200 | r = gmc_v8_0_tonga_mc_load_microcode(adev); |
1201 | if (r) { |
1202 | DRM_ERROR("Failed to load MC firmware!\n" ); |
1203 | return r; |
1204 | } |
1205 | } else if (adev->asic_type == CHIP_POLARIS11 || |
1206 | adev->asic_type == CHIP_POLARIS10 || |
1207 | adev->asic_type == CHIP_POLARIS12) { |
1208 | r = gmc_v8_0_polaris_mc_load_microcode(adev); |
1209 | if (r) { |
1210 | DRM_ERROR("Failed to load MC firmware!\n" ); |
1211 | return r; |
1212 | } |
1213 | } |
1214 | |
1215 | r = gmc_v8_0_gart_enable(adev); |
1216 | if (r) |
1217 | return r; |
1218 | |
1219 | if (amdgpu_emu_mode == 1) |
1220 | return amdgpu_gmc_vram_checking(adev); |
1221 | else |
1222 | return r; |
1223 | } |
1224 | |
1225 | static int gmc_v8_0_hw_fini(void *handle) |
1226 | { |
1227 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1228 | |
1229 | amdgpu_irq_put(adev, src: &adev->gmc.vm_fault, type: 0); |
1230 | gmc_v8_0_gart_disable(adev); |
1231 | |
1232 | return 0; |
1233 | } |
1234 | |
1235 | static int gmc_v8_0_suspend(void *handle) |
1236 | { |
1237 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1238 | |
1239 | gmc_v8_0_hw_fini(handle: adev); |
1240 | |
1241 | return 0; |
1242 | } |
1243 | |
1244 | static int gmc_v8_0_resume(void *handle) |
1245 | { |
1246 | int r; |
1247 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1248 | |
1249 | r = gmc_v8_0_hw_init(handle: adev); |
1250 | if (r) |
1251 | return r; |
1252 | |
1253 | amdgpu_vmid_reset_all(adev); |
1254 | |
1255 | return 0; |
1256 | } |
1257 | |
1258 | static bool gmc_v8_0_is_idle(void *handle) |
1259 | { |
1260 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1261 | u32 tmp = RREG32(mmSRBM_STATUS); |
1262 | |
1263 | if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | |
1264 | SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK)) |
1265 | return false; |
1266 | |
1267 | return true; |
1268 | } |
1269 | |
1270 | static int gmc_v8_0_wait_for_idle(void *handle) |
1271 | { |
1272 | unsigned int i; |
1273 | u32 tmp; |
1274 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1275 | |
1276 | for (i = 0; i < adev->usec_timeout; i++) { |
1277 | /* read MC_STATUS */ |
1278 | tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK | |
1279 | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | |
1280 | SRBM_STATUS__MCC_BUSY_MASK | |
1281 | SRBM_STATUS__MCD_BUSY_MASK | |
1282 | SRBM_STATUS__VMC_BUSY_MASK | |
1283 | SRBM_STATUS__VMC1_BUSY_MASK); |
1284 | if (!tmp) |
1285 | return 0; |
1286 | udelay(1); |
1287 | } |
1288 | return -ETIMEDOUT; |
1289 | |
1290 | } |
1291 | |
1292 | static bool gmc_v8_0_check_soft_reset(void *handle) |
1293 | { |
1294 | u32 srbm_soft_reset = 0; |
1295 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1296 | u32 tmp = RREG32(mmSRBM_STATUS); |
1297 | |
1298 | if (tmp & SRBM_STATUS__VMC_BUSY_MASK) |
1299 | srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, |
1300 | SRBM_SOFT_RESET, SOFT_RESET_VMC, 1); |
1301 | |
1302 | if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | |
1303 | SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) { |
1304 | if (!(adev->flags & AMD_IS_APU)) |
1305 | srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, |
1306 | SRBM_SOFT_RESET, SOFT_RESET_MC, 1); |
1307 | } |
1308 | |
1309 | if (srbm_soft_reset) { |
1310 | adev->gmc.srbm_soft_reset = srbm_soft_reset; |
1311 | return true; |
1312 | } |
1313 | |
1314 | adev->gmc.srbm_soft_reset = 0; |
1315 | |
1316 | return false; |
1317 | } |
1318 | |
1319 | static int gmc_v8_0_pre_soft_reset(void *handle) |
1320 | { |
1321 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1322 | |
1323 | if (!adev->gmc.srbm_soft_reset) |
1324 | return 0; |
1325 | |
1326 | gmc_v8_0_mc_stop(adev); |
1327 | if (gmc_v8_0_wait_for_idle(handle: adev)) |
1328 | dev_warn(adev->dev, "Wait for GMC idle timed out !\n" ); |
1329 | |
1330 | return 0; |
1331 | } |
1332 | |
1333 | static int gmc_v8_0_soft_reset(void *handle) |
1334 | { |
1335 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1336 | u32 srbm_soft_reset; |
1337 | |
1338 | if (!adev->gmc.srbm_soft_reset) |
1339 | return 0; |
1340 | srbm_soft_reset = adev->gmc.srbm_soft_reset; |
1341 | |
1342 | if (srbm_soft_reset) { |
1343 | u32 tmp; |
1344 | |
1345 | tmp = RREG32(mmSRBM_SOFT_RESET); |
1346 | tmp |= srbm_soft_reset; |
1347 | dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n" , tmp); |
1348 | WREG32(mmSRBM_SOFT_RESET, tmp); |
1349 | tmp = RREG32(mmSRBM_SOFT_RESET); |
1350 | |
1351 | udelay(50); |
1352 | |
1353 | tmp &= ~srbm_soft_reset; |
1354 | WREG32(mmSRBM_SOFT_RESET, tmp); |
1355 | tmp = RREG32(mmSRBM_SOFT_RESET); |
1356 | |
1357 | /* Wait a little for things to settle down */ |
1358 | udelay(50); |
1359 | } |
1360 | |
1361 | return 0; |
1362 | } |
1363 | |
1364 | static int gmc_v8_0_post_soft_reset(void *handle) |
1365 | { |
1366 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1367 | |
1368 | if (!adev->gmc.srbm_soft_reset) |
1369 | return 0; |
1370 | |
1371 | gmc_v8_0_mc_resume(adev); |
1372 | return 0; |
1373 | } |
1374 | |
1375 | static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev, |
1376 | struct amdgpu_irq_src *src, |
1377 | unsigned int type, |
1378 | enum amdgpu_interrupt_state state) |
1379 | { |
1380 | u32 tmp; |
1381 | u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | |
1382 | VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | |
1383 | VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | |
1384 | VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | |
1385 | VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | |
1386 | VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | |
1387 | VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK); |
1388 | |
1389 | switch (state) { |
1390 | case AMDGPU_IRQ_STATE_DISABLE: |
1391 | /* system context */ |
1392 | tmp = RREG32(mmVM_CONTEXT0_CNTL); |
1393 | tmp &= ~bits; |
1394 | WREG32(mmVM_CONTEXT0_CNTL, tmp); |
1395 | /* VMs */ |
1396 | tmp = RREG32(mmVM_CONTEXT1_CNTL); |
1397 | tmp &= ~bits; |
1398 | WREG32(mmVM_CONTEXT1_CNTL, tmp); |
1399 | break; |
1400 | case AMDGPU_IRQ_STATE_ENABLE: |
1401 | /* system context */ |
1402 | tmp = RREG32(mmVM_CONTEXT0_CNTL); |
1403 | tmp |= bits; |
1404 | WREG32(mmVM_CONTEXT0_CNTL, tmp); |
1405 | /* VMs */ |
1406 | tmp = RREG32(mmVM_CONTEXT1_CNTL); |
1407 | tmp |= bits; |
1408 | WREG32(mmVM_CONTEXT1_CNTL, tmp); |
1409 | break; |
1410 | default: |
1411 | break; |
1412 | } |
1413 | |
1414 | return 0; |
1415 | } |
1416 | |
1417 | static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev, |
1418 | struct amdgpu_irq_src *source, |
1419 | struct amdgpu_iv_entry *entry) |
1420 | { |
1421 | u32 addr, status, mc_client, vmid; |
1422 | |
1423 | if (amdgpu_sriov_vf(adev)) { |
1424 | dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n" , |
1425 | entry->src_id, entry->src_data[0]); |
1426 | dev_err(adev->dev, " Can't decode VM fault info here on SRIOV VF\n" ); |
1427 | return 0; |
1428 | } |
1429 | |
1430 | addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR); |
1431 | status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS); |
1432 | mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT); |
1433 | /* reset addr and status */ |
1434 | WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1); |
1435 | |
1436 | if (!addr && !status) |
1437 | return 0; |
1438 | |
1439 | amdgpu_vm_update_fault_cache(adev, pasid: entry->pasid, |
1440 | addr: ((u64)addr) << AMDGPU_GPU_PAGE_SHIFT, status, AMDGPU_GFXHUB(0)); |
1441 | |
1442 | if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST) |
1443 | gmc_v8_0_set_fault_enable_default(adev, value: false); |
1444 | |
1445 | if (printk_ratelimit()) { |
1446 | struct amdgpu_task_info task_info; |
1447 | |
1448 | memset(&task_info, 0, sizeof(struct amdgpu_task_info)); |
1449 | amdgpu_vm_get_task_info(adev, pasid: entry->pasid, task_info: &task_info); |
1450 | |
1451 | dev_err(adev->dev, "GPU fault detected: %d 0x%08x for process %s pid %d thread %s pid %d\n" , |
1452 | entry->src_id, entry->src_data[0], task_info.process_name, |
1453 | task_info.tgid, task_info.task_name, task_info.pid); |
1454 | dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n" , |
1455 | addr); |
1456 | dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n" , |
1457 | status); |
1458 | gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client, |
1459 | pasid: entry->pasid); |
1460 | } |
1461 | |
1462 | vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, |
1463 | VMID); |
1464 | if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid) |
1465 | && !atomic_read(v: &adev->gmc.vm_fault_info_updated)) { |
1466 | struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info; |
1467 | u32 protections = REG_GET_FIELD(status, |
1468 | VM_CONTEXT1_PROTECTION_FAULT_STATUS, |
1469 | PROTECTIONS); |
1470 | |
1471 | info->vmid = vmid; |
1472 | info->mc_id = REG_GET_FIELD(status, |
1473 | VM_CONTEXT1_PROTECTION_FAULT_STATUS, |
1474 | MEMORY_CLIENT_ID); |
1475 | info->status = status; |
1476 | info->page_addr = addr; |
1477 | info->prot_valid = protections & 0x7 ? true : false; |
1478 | info->prot_read = protections & 0x8 ? true : false; |
1479 | info->prot_write = protections & 0x10 ? true : false; |
1480 | info->prot_exec = protections & 0x20 ? true : false; |
1481 | mb(); |
1482 | atomic_set(v: &adev->gmc.vm_fault_info_updated, i: 1); |
1483 | } |
1484 | |
1485 | return 0; |
1486 | } |
1487 | |
1488 | static void fiji_update_mc_medium_grain_clock_gating(struct amdgpu_device *adev, |
1489 | bool enable) |
1490 | { |
1491 | uint32_t data; |
1492 | |
1493 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) { |
1494 | data = RREG32(mmMC_HUB_MISC_HUB_CG); |
1495 | data |= MC_HUB_MISC_HUB_CG__ENABLE_MASK; |
1496 | WREG32(mmMC_HUB_MISC_HUB_CG, data); |
1497 | |
1498 | data = RREG32(mmMC_HUB_MISC_SIP_CG); |
1499 | data |= MC_HUB_MISC_SIP_CG__ENABLE_MASK; |
1500 | WREG32(mmMC_HUB_MISC_SIP_CG, data); |
1501 | |
1502 | data = RREG32(mmMC_HUB_MISC_VM_CG); |
1503 | data |= MC_HUB_MISC_VM_CG__ENABLE_MASK; |
1504 | WREG32(mmMC_HUB_MISC_VM_CG, data); |
1505 | |
1506 | data = RREG32(mmMC_XPB_CLK_GAT); |
1507 | data |= MC_XPB_CLK_GAT__ENABLE_MASK; |
1508 | WREG32(mmMC_XPB_CLK_GAT, data); |
1509 | |
1510 | data = RREG32(mmATC_MISC_CG); |
1511 | data |= ATC_MISC_CG__ENABLE_MASK; |
1512 | WREG32(mmATC_MISC_CG, data); |
1513 | |
1514 | data = RREG32(mmMC_CITF_MISC_WR_CG); |
1515 | data |= MC_CITF_MISC_WR_CG__ENABLE_MASK; |
1516 | WREG32(mmMC_CITF_MISC_WR_CG, data); |
1517 | |
1518 | data = RREG32(mmMC_CITF_MISC_RD_CG); |
1519 | data |= MC_CITF_MISC_RD_CG__ENABLE_MASK; |
1520 | WREG32(mmMC_CITF_MISC_RD_CG, data); |
1521 | |
1522 | data = RREG32(mmMC_CITF_MISC_VM_CG); |
1523 | data |= MC_CITF_MISC_VM_CG__ENABLE_MASK; |
1524 | WREG32(mmMC_CITF_MISC_VM_CG, data); |
1525 | |
1526 | data = RREG32(mmVM_L2_CG); |
1527 | data |= VM_L2_CG__ENABLE_MASK; |
1528 | WREG32(mmVM_L2_CG, data); |
1529 | } else { |
1530 | data = RREG32(mmMC_HUB_MISC_HUB_CG); |
1531 | data &= ~MC_HUB_MISC_HUB_CG__ENABLE_MASK; |
1532 | WREG32(mmMC_HUB_MISC_HUB_CG, data); |
1533 | |
1534 | data = RREG32(mmMC_HUB_MISC_SIP_CG); |
1535 | data &= ~MC_HUB_MISC_SIP_CG__ENABLE_MASK; |
1536 | WREG32(mmMC_HUB_MISC_SIP_CG, data); |
1537 | |
1538 | data = RREG32(mmMC_HUB_MISC_VM_CG); |
1539 | data &= ~MC_HUB_MISC_VM_CG__ENABLE_MASK; |
1540 | WREG32(mmMC_HUB_MISC_VM_CG, data); |
1541 | |
1542 | data = RREG32(mmMC_XPB_CLK_GAT); |
1543 | data &= ~MC_XPB_CLK_GAT__ENABLE_MASK; |
1544 | WREG32(mmMC_XPB_CLK_GAT, data); |
1545 | |
1546 | data = RREG32(mmATC_MISC_CG); |
1547 | data &= ~ATC_MISC_CG__ENABLE_MASK; |
1548 | WREG32(mmATC_MISC_CG, data); |
1549 | |
1550 | data = RREG32(mmMC_CITF_MISC_WR_CG); |
1551 | data &= ~MC_CITF_MISC_WR_CG__ENABLE_MASK; |
1552 | WREG32(mmMC_CITF_MISC_WR_CG, data); |
1553 | |
1554 | data = RREG32(mmMC_CITF_MISC_RD_CG); |
1555 | data &= ~MC_CITF_MISC_RD_CG__ENABLE_MASK; |
1556 | WREG32(mmMC_CITF_MISC_RD_CG, data); |
1557 | |
1558 | data = RREG32(mmMC_CITF_MISC_VM_CG); |
1559 | data &= ~MC_CITF_MISC_VM_CG__ENABLE_MASK; |
1560 | WREG32(mmMC_CITF_MISC_VM_CG, data); |
1561 | |
1562 | data = RREG32(mmVM_L2_CG); |
1563 | data &= ~VM_L2_CG__ENABLE_MASK; |
1564 | WREG32(mmVM_L2_CG, data); |
1565 | } |
1566 | } |
1567 | |
1568 | static void fiji_update_mc_light_sleep(struct amdgpu_device *adev, |
1569 | bool enable) |
1570 | { |
1571 | uint32_t data; |
1572 | |
1573 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) { |
1574 | data = RREG32(mmMC_HUB_MISC_HUB_CG); |
1575 | data |= MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK; |
1576 | WREG32(mmMC_HUB_MISC_HUB_CG, data); |
1577 | |
1578 | data = RREG32(mmMC_HUB_MISC_SIP_CG); |
1579 | data |= MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK; |
1580 | WREG32(mmMC_HUB_MISC_SIP_CG, data); |
1581 | |
1582 | data = RREG32(mmMC_HUB_MISC_VM_CG); |
1583 | data |= MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK; |
1584 | WREG32(mmMC_HUB_MISC_VM_CG, data); |
1585 | |
1586 | data = RREG32(mmMC_XPB_CLK_GAT); |
1587 | data |= MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK; |
1588 | WREG32(mmMC_XPB_CLK_GAT, data); |
1589 | |
1590 | data = RREG32(mmATC_MISC_CG); |
1591 | data |= ATC_MISC_CG__MEM_LS_ENABLE_MASK; |
1592 | WREG32(mmATC_MISC_CG, data); |
1593 | |
1594 | data = RREG32(mmMC_CITF_MISC_WR_CG); |
1595 | data |= MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK; |
1596 | WREG32(mmMC_CITF_MISC_WR_CG, data); |
1597 | |
1598 | data = RREG32(mmMC_CITF_MISC_RD_CG); |
1599 | data |= MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK; |
1600 | WREG32(mmMC_CITF_MISC_RD_CG, data); |
1601 | |
1602 | data = RREG32(mmMC_CITF_MISC_VM_CG); |
1603 | data |= MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK; |
1604 | WREG32(mmMC_CITF_MISC_VM_CG, data); |
1605 | |
1606 | data = RREG32(mmVM_L2_CG); |
1607 | data |= VM_L2_CG__MEM_LS_ENABLE_MASK; |
1608 | WREG32(mmVM_L2_CG, data); |
1609 | } else { |
1610 | data = RREG32(mmMC_HUB_MISC_HUB_CG); |
1611 | data &= ~MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK; |
1612 | WREG32(mmMC_HUB_MISC_HUB_CG, data); |
1613 | |
1614 | data = RREG32(mmMC_HUB_MISC_SIP_CG); |
1615 | data &= ~MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK; |
1616 | WREG32(mmMC_HUB_MISC_SIP_CG, data); |
1617 | |
1618 | data = RREG32(mmMC_HUB_MISC_VM_CG); |
1619 | data &= ~MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK; |
1620 | WREG32(mmMC_HUB_MISC_VM_CG, data); |
1621 | |
1622 | data = RREG32(mmMC_XPB_CLK_GAT); |
1623 | data &= ~MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK; |
1624 | WREG32(mmMC_XPB_CLK_GAT, data); |
1625 | |
1626 | data = RREG32(mmATC_MISC_CG); |
1627 | data &= ~ATC_MISC_CG__MEM_LS_ENABLE_MASK; |
1628 | WREG32(mmATC_MISC_CG, data); |
1629 | |
1630 | data = RREG32(mmMC_CITF_MISC_WR_CG); |
1631 | data &= ~MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK; |
1632 | WREG32(mmMC_CITF_MISC_WR_CG, data); |
1633 | |
1634 | data = RREG32(mmMC_CITF_MISC_RD_CG); |
1635 | data &= ~MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK; |
1636 | WREG32(mmMC_CITF_MISC_RD_CG, data); |
1637 | |
1638 | data = RREG32(mmMC_CITF_MISC_VM_CG); |
1639 | data &= ~MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK; |
1640 | WREG32(mmMC_CITF_MISC_VM_CG, data); |
1641 | |
1642 | data = RREG32(mmVM_L2_CG); |
1643 | data &= ~VM_L2_CG__MEM_LS_ENABLE_MASK; |
1644 | WREG32(mmVM_L2_CG, data); |
1645 | } |
1646 | } |
1647 | |
1648 | static int gmc_v8_0_set_clockgating_state(void *handle, |
1649 | enum amd_clockgating_state state) |
1650 | { |
1651 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1652 | |
1653 | if (amdgpu_sriov_vf(adev)) |
1654 | return 0; |
1655 | |
1656 | switch (adev->asic_type) { |
1657 | case CHIP_FIJI: |
1658 | fiji_update_mc_medium_grain_clock_gating(adev, |
1659 | enable: state == AMD_CG_STATE_GATE); |
1660 | fiji_update_mc_light_sleep(adev, |
1661 | enable: state == AMD_CG_STATE_GATE); |
1662 | break; |
1663 | default: |
1664 | break; |
1665 | } |
1666 | return 0; |
1667 | } |
1668 | |
1669 | static int gmc_v8_0_set_powergating_state(void *handle, |
1670 | enum amd_powergating_state state) |
1671 | { |
1672 | return 0; |
1673 | } |
1674 | |
1675 | static void gmc_v8_0_get_clockgating_state(void *handle, u64 *flags) |
1676 | { |
1677 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
1678 | int data; |
1679 | |
1680 | if (amdgpu_sriov_vf(adev)) |
1681 | *flags = 0; |
1682 | |
1683 | /* AMD_CG_SUPPORT_MC_MGCG */ |
1684 | data = RREG32(mmMC_HUB_MISC_HUB_CG); |
1685 | if (data & MC_HUB_MISC_HUB_CG__ENABLE_MASK) |
1686 | *flags |= AMD_CG_SUPPORT_MC_MGCG; |
1687 | |
1688 | /* AMD_CG_SUPPORT_MC_LS */ |
1689 | if (data & MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK) |
1690 | *flags |= AMD_CG_SUPPORT_MC_LS; |
1691 | } |
1692 | |
1693 | static const struct amd_ip_funcs gmc_v8_0_ip_funcs = { |
1694 | .name = "gmc_v8_0" , |
1695 | .early_init = gmc_v8_0_early_init, |
1696 | .late_init = gmc_v8_0_late_init, |
1697 | .sw_init = gmc_v8_0_sw_init, |
1698 | .sw_fini = gmc_v8_0_sw_fini, |
1699 | .hw_init = gmc_v8_0_hw_init, |
1700 | .hw_fini = gmc_v8_0_hw_fini, |
1701 | .suspend = gmc_v8_0_suspend, |
1702 | .resume = gmc_v8_0_resume, |
1703 | .is_idle = gmc_v8_0_is_idle, |
1704 | .wait_for_idle = gmc_v8_0_wait_for_idle, |
1705 | .check_soft_reset = gmc_v8_0_check_soft_reset, |
1706 | .pre_soft_reset = gmc_v8_0_pre_soft_reset, |
1707 | .soft_reset = gmc_v8_0_soft_reset, |
1708 | .post_soft_reset = gmc_v8_0_post_soft_reset, |
1709 | .set_clockgating_state = gmc_v8_0_set_clockgating_state, |
1710 | .set_powergating_state = gmc_v8_0_set_powergating_state, |
1711 | .get_clockgating_state = gmc_v8_0_get_clockgating_state, |
1712 | }; |
1713 | |
1714 | static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = { |
1715 | .flush_gpu_tlb = gmc_v8_0_flush_gpu_tlb, |
1716 | .flush_gpu_tlb_pasid = gmc_v8_0_flush_gpu_tlb_pasid, |
1717 | .emit_flush_gpu_tlb = gmc_v8_0_emit_flush_gpu_tlb, |
1718 | .emit_pasid_mapping = gmc_v8_0_emit_pasid_mapping, |
1719 | .set_prt = gmc_v8_0_set_prt, |
1720 | .get_vm_pde = gmc_v8_0_get_vm_pde, |
1721 | .get_vm_pte = gmc_v8_0_get_vm_pte, |
1722 | .get_vbios_fb_size = gmc_v8_0_get_vbios_fb_size, |
1723 | }; |
1724 | |
1725 | static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs = { |
1726 | .set = gmc_v8_0_vm_fault_interrupt_state, |
1727 | .process = gmc_v8_0_process_interrupt, |
1728 | }; |
1729 | |
1730 | static void gmc_v8_0_set_gmc_funcs(struct amdgpu_device *adev) |
1731 | { |
1732 | adev->gmc.gmc_funcs = &gmc_v8_0_gmc_funcs; |
1733 | } |
1734 | |
1735 | static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev) |
1736 | { |
1737 | adev->gmc.vm_fault.num_types = 1; |
1738 | adev->gmc.vm_fault.funcs = &gmc_v8_0_irq_funcs; |
1739 | } |
1740 | |
1741 | const struct amdgpu_ip_block_version gmc_v8_0_ip_block = { |
1742 | .type = AMD_IP_BLOCK_TYPE_GMC, |
1743 | .major = 8, |
1744 | .minor = 0, |
1745 | .rev = 0, |
1746 | .funcs = &gmc_v8_0_ip_funcs, |
1747 | }; |
1748 | |
1749 | const struct amdgpu_ip_block_version gmc_v8_1_ip_block = { |
1750 | .type = AMD_IP_BLOCK_TYPE_GMC, |
1751 | .major = 8, |
1752 | .minor = 1, |
1753 | .rev = 0, |
1754 | .funcs = &gmc_v8_0_ip_funcs, |
1755 | }; |
1756 | |
1757 | const struct amdgpu_ip_block_version gmc_v8_5_ip_block = { |
1758 | .type = AMD_IP_BLOCK_TYPE_GMC, |
1759 | .major = 8, |
1760 | .minor = 5, |
1761 | .rev = 0, |
1762 | .funcs = &gmc_v8_0_ip_funcs, |
1763 | }; |
1764 | |