1 | /* |
2 | * Copyright 2022 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | #include "amdgpu.h" |
24 | #include "mmhub_v1_8.h" |
25 | |
26 | #include "mmhub/mmhub_1_8_0_offset.h" |
27 | #include "mmhub/mmhub_1_8_0_sh_mask.h" |
28 | #include "vega10_enum.h" |
29 | |
30 | #include "soc15_common.h" |
31 | #include "soc15.h" |
32 | #include "amdgpu_ras.h" |
33 | |
34 | #define regVM_L2_CNTL3_DEFAULT 0x80100007 |
35 | #define regVM_L2_CNTL4_DEFAULT 0x000000c1 |
36 | #define mmSMNAID_AID0_MCA_SMU 0x03b30400 |
37 | |
38 | static u64 mmhub_v1_8_get_fb_location(struct amdgpu_device *adev) |
39 | { |
40 | u64 base = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE); |
41 | u64 top = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP); |
42 | |
43 | base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK; |
44 | base <<= 24; |
45 | |
46 | top &= MC_VM_FB_LOCATION_TOP__FB_TOP_MASK; |
47 | top <<= 24; |
48 | |
49 | adev->gmc.fb_start = base; |
50 | adev->gmc.fb_end = top; |
51 | |
52 | return base; |
53 | } |
54 | |
55 | static void mmhub_v1_8_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, |
56 | uint64_t page_table_base) |
57 | { |
58 | struct amdgpu_vmhub *hub; |
59 | u32 inst_mask; |
60 | int i; |
61 | |
62 | inst_mask = adev->aid_mask; |
63 | for_each_inst(i, inst_mask) { |
64 | hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; |
65 | WREG32_SOC15_OFFSET(MMHUB, i, |
66 | regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, |
67 | hub->ctx_addr_distance * vmid, |
68 | lower_32_bits(page_table_base)); |
69 | |
70 | WREG32_SOC15_OFFSET(MMHUB, i, |
71 | regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, |
72 | hub->ctx_addr_distance * vmid, |
73 | upper_32_bits(page_table_base)); |
74 | } |
75 | } |
76 | |
77 | static void mmhub_v1_8_init_gart_aperture_regs(struct amdgpu_device *adev) |
78 | { |
79 | uint64_t pt_base; |
80 | u32 inst_mask; |
81 | int i; |
82 | |
83 | if (adev->gmc.pdb0_bo) |
84 | pt_base = amdgpu_gmc_pd_addr(bo: adev->gmc.pdb0_bo); |
85 | else |
86 | pt_base = amdgpu_gmc_pd_addr(bo: adev->gart.bo); |
87 | |
88 | mmhub_v1_8_setup_vm_pt_regs(adev, vmid: 0, page_table_base: pt_base); |
89 | |
90 | /* If use GART for FB translation, vmid0 page table covers both |
91 | * vram and system memory (gart) |
92 | */ |
93 | inst_mask = adev->aid_mask; |
94 | for_each_inst(i, inst_mask) { |
95 | if (adev->gmc.pdb0_bo) { |
96 | WREG32_SOC15(MMHUB, i, |
97 | regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, |
98 | (u32)(adev->gmc.fb_start >> 12)); |
99 | WREG32_SOC15(MMHUB, i, |
100 | regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, |
101 | (u32)(adev->gmc.fb_start >> 44)); |
102 | |
103 | WREG32_SOC15(MMHUB, i, |
104 | regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, |
105 | (u32)(adev->gmc.gart_end >> 12)); |
106 | WREG32_SOC15(MMHUB, i, |
107 | regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, |
108 | (u32)(adev->gmc.gart_end >> 44)); |
109 | |
110 | } else { |
111 | WREG32_SOC15(MMHUB, i, |
112 | regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, |
113 | (u32)(adev->gmc.gart_start >> 12)); |
114 | WREG32_SOC15(MMHUB, i, |
115 | regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, |
116 | (u32)(adev->gmc.gart_start >> 44)); |
117 | |
118 | WREG32_SOC15(MMHUB, i, |
119 | regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, |
120 | (u32)(adev->gmc.gart_end >> 12)); |
121 | WREG32_SOC15(MMHUB, i, |
122 | regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, |
123 | (u32)(adev->gmc.gart_end >> 44)); |
124 | } |
125 | } |
126 | } |
127 | |
128 | static void mmhub_v1_8_init_system_aperture_regs(struct amdgpu_device *adev) |
129 | { |
130 | uint32_t tmp, inst_mask; |
131 | uint64_t value; |
132 | int i; |
133 | |
134 | if (amdgpu_sriov_vf(adev)) |
135 | return; |
136 | |
137 | inst_mask = adev->aid_mask; |
138 | for_each_inst(i, inst_mask) { |
139 | /* Program the AGP BAR */ |
140 | WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BASE, 0); |
141 | WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, |
142 | adev->gmc.agp_start >> 24); |
143 | WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, |
144 | adev->gmc.agp_end >> 24); |
145 | |
146 | /* Program the system aperture low logical page number. */ |
147 | WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_LOW_ADDR, |
148 | min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18); |
149 | |
150 | WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, |
151 | max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18); |
152 | |
153 | /* In the case squeezing vram into GART aperture, we don't use |
154 | * FB aperture and AGP aperture. Disable them. |
155 | */ |
156 | if (adev->gmc.pdb0_bo) { |
157 | WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, 0xFFFFFF); |
158 | WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, 0); |
159 | WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_TOP, 0); |
160 | WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_BASE, |
161 | 0x00FFFFFF); |
162 | WREG32_SOC15(MMHUB, i, |
163 | regMC_VM_SYSTEM_APERTURE_LOW_ADDR, |
164 | 0x3FFFFFFF); |
165 | WREG32_SOC15(MMHUB, i, |
166 | regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0); |
167 | } |
168 | |
169 | /* Set default page address. */ |
170 | value = amdgpu_gmc_vram_mc2pa(adev, mc_addr: adev->mem_scratch.gpu_addr); |
171 | WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, |
172 | (u32)(value >> 12)); |
173 | WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, |
174 | (u32)(value >> 44)); |
175 | |
176 | /* Program "protection fault". */ |
177 | WREG32_SOC15(MMHUB, i, |
178 | regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, |
179 | (u32)(adev->dummy_page_addr >> 12)); |
180 | WREG32_SOC15(MMHUB, i, |
181 | regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, |
182 | (u32)((u64)adev->dummy_page_addr >> 44)); |
183 | |
184 | tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2); |
185 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2, |
186 | ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); |
187 | WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2, tmp); |
188 | } |
189 | } |
190 | |
191 | static void mmhub_v1_8_init_tlb_regs(struct amdgpu_device *adev) |
192 | { |
193 | uint32_t tmp, inst_mask; |
194 | int i; |
195 | |
196 | /* Setup TLB control */ |
197 | inst_mask = adev->aid_mask; |
198 | for_each_inst(i, inst_mask) { |
199 | tmp = RREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL); |
200 | |
201 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, |
202 | 1); |
203 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, |
204 | SYSTEM_ACCESS_MODE, 3); |
205 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, |
206 | ENABLE_ADVANCED_DRIVER_MODEL, 1); |
207 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, |
208 | SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); |
209 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, |
210 | MTYPE, MTYPE_UC);/* XXX for emulation. */ |
211 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1); |
212 | |
213 | WREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL, tmp); |
214 | } |
215 | } |
216 | |
217 | static void mmhub_v1_8_init_cache_regs(struct amdgpu_device *adev) |
218 | { |
219 | uint32_t tmp, inst_mask; |
220 | int i; |
221 | |
222 | if (amdgpu_sriov_vf(adev)) |
223 | return; |
224 | |
225 | /* Setup L2 cache */ |
226 | inst_mask = adev->aid_mask; |
227 | for_each_inst(i, inst_mask) { |
228 | tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL); |
229 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1); |
230 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, |
231 | ENABLE_L2_FRAGMENT_PROCESSING, 1); |
232 | /* XXX for emulation, Refer to closed source code.*/ |
233 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, |
234 | L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); |
235 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, |
236 | 0); |
237 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, |
238 | CONTEXT1_IDENTITY_ACCESS_MODE, 1); |
239 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, |
240 | IDENTITY_MODE_FRAGMENT_SIZE, 0); |
241 | WREG32_SOC15(MMHUB, i, regVM_L2_CNTL, tmp); |
242 | |
243 | tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL2); |
244 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, |
245 | 1); |
246 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); |
247 | WREG32_SOC15(MMHUB, i, regVM_L2_CNTL2, tmp); |
248 | |
249 | tmp = regVM_L2_CNTL3_DEFAULT; |
250 | if (adev->gmc.translate_further) { |
251 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12); |
252 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, |
253 | L2_CACHE_BIGK_FRAGMENT_SIZE, 9); |
254 | } else { |
255 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9); |
256 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, |
257 | L2_CACHE_BIGK_FRAGMENT_SIZE, 6); |
258 | } |
259 | WREG32_SOC15(MMHUB, i, regVM_L2_CNTL3, tmp); |
260 | |
261 | tmp = regVM_L2_CNTL4_DEFAULT; |
262 | /* For AMD APP APUs setup WC memory */ |
263 | if (adev->gmc.xgmi.connected_to_cpu || adev->gmc.is_app_apu) { |
264 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, |
265 | VMC_TAP_PDE_REQUEST_PHYSICAL, 1); |
266 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, |
267 | VMC_TAP_PTE_REQUEST_PHYSICAL, 1); |
268 | } else { |
269 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, |
270 | VMC_TAP_PDE_REQUEST_PHYSICAL, 0); |
271 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, |
272 | VMC_TAP_PTE_REQUEST_PHYSICAL, 0); |
273 | } |
274 | WREG32_SOC15(MMHUB, i, regVM_L2_CNTL4, tmp); |
275 | } |
276 | } |
277 | |
278 | static void mmhub_v1_8_enable_system_domain(struct amdgpu_device *adev) |
279 | { |
280 | uint32_t tmp, inst_mask; |
281 | int i; |
282 | |
283 | inst_mask = adev->aid_mask; |
284 | for_each_inst(i, inst_mask) { |
285 | tmp = RREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL); |
286 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); |
287 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, |
288 | adev->gmc.vmid0_page_table_depth); |
289 | tmp = REG_SET_FIELD(tmp, |
290 | VM_CONTEXT0_CNTL, PAGE_TABLE_BLOCK_SIZE, |
291 | adev->gmc.vmid0_page_table_block_size); |
292 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, |
293 | RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); |
294 | WREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL, tmp); |
295 | } |
296 | } |
297 | |
298 | static void mmhub_v1_8_disable_identity_aperture(struct amdgpu_device *adev) |
299 | { |
300 | u32 inst_mask; |
301 | int i; |
302 | |
303 | if (amdgpu_sriov_vf(adev)) |
304 | return; |
305 | |
306 | inst_mask = adev->aid_mask; |
307 | for_each_inst(i, inst_mask) { |
308 | WREG32_SOC15(MMHUB, i, |
309 | regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, |
310 | 0XFFFFFFFF); |
311 | WREG32_SOC15(MMHUB, i, |
312 | regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, |
313 | 0x0000000F); |
314 | |
315 | WREG32_SOC15(MMHUB, i, |
316 | regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, |
317 | 0); |
318 | WREG32_SOC15(MMHUB, i, |
319 | regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, |
320 | 0); |
321 | |
322 | WREG32_SOC15(MMHUB, i, |
323 | regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0); |
324 | WREG32_SOC15(MMHUB, i, |
325 | regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0); |
326 | } |
327 | } |
328 | |
329 | static void mmhub_v1_8_setup_vmid_config(struct amdgpu_device *adev) |
330 | { |
331 | struct amdgpu_vmhub *hub; |
332 | unsigned int num_level, block_size; |
333 | uint32_t tmp, inst_mask; |
334 | int i, j; |
335 | |
336 | num_level = adev->vm_manager.num_level; |
337 | block_size = adev->vm_manager.block_size; |
338 | if (adev->gmc.translate_further) |
339 | num_level -= 1; |
340 | else |
341 | block_size -= 9; |
342 | |
343 | inst_mask = adev->aid_mask; |
344 | for_each_inst(j, inst_mask) { |
345 | hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; |
346 | for (i = 0; i <= 14; i++) { |
347 | tmp = RREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, |
348 | i * hub->ctx_distance); |
349 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
350 | ENABLE_CONTEXT, 1); |
351 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
352 | PAGE_TABLE_DEPTH, num_level); |
353 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
354 | RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
355 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
356 | DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
357 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
358 | PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
359 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
360 | VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
361 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
362 | READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
363 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
364 | WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
365 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
366 | EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); |
367 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
368 | PAGE_TABLE_BLOCK_SIZE, |
369 | block_size); |
370 | /* On 9.4.3, XNACK can be enabled in the SQ |
371 | * per-process. Retry faults need to be enabled for |
372 | * that to work. |
373 | */ |
374 | tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, |
375 | RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 1); |
376 | WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, |
377 | i * hub->ctx_distance, tmp); |
378 | WREG32_SOC15_OFFSET(MMHUB, j, |
379 | regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, |
380 | i * hub->ctx_addr_distance, 0); |
381 | WREG32_SOC15_OFFSET(MMHUB, j, |
382 | regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, |
383 | i * hub->ctx_addr_distance, 0); |
384 | WREG32_SOC15_OFFSET(MMHUB, j, |
385 | regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, |
386 | i * hub->ctx_addr_distance, |
387 | lower_32_bits(adev->vm_manager.max_pfn - 1)); |
388 | WREG32_SOC15_OFFSET(MMHUB, j, |
389 | regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, |
390 | i * hub->ctx_addr_distance, |
391 | upper_32_bits(adev->vm_manager.max_pfn - 1)); |
392 | } |
393 | } |
394 | } |
395 | |
396 | static void mmhub_v1_8_program_invalidation(struct amdgpu_device *adev) |
397 | { |
398 | struct amdgpu_vmhub *hub; |
399 | u32 i, j, inst_mask; |
400 | |
401 | inst_mask = adev->aid_mask; |
402 | for_each_inst(j, inst_mask) { |
403 | hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; |
404 | for (i = 0; i < 18; ++i) { |
405 | WREG32_SOC15_OFFSET(MMHUB, j, |
406 | regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, |
407 | i * hub->eng_addr_distance, 0xffffffff); |
408 | WREG32_SOC15_OFFSET(MMHUB, j, |
409 | regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, |
410 | i * hub->eng_addr_distance, 0x1f); |
411 | } |
412 | } |
413 | } |
414 | |
415 | static int mmhub_v1_8_gart_enable(struct amdgpu_device *adev) |
416 | { |
417 | /* GART Enable. */ |
418 | mmhub_v1_8_init_gart_aperture_regs(adev); |
419 | mmhub_v1_8_init_system_aperture_regs(adev); |
420 | mmhub_v1_8_init_tlb_regs(adev); |
421 | mmhub_v1_8_init_cache_regs(adev); |
422 | |
423 | mmhub_v1_8_enable_system_domain(adev); |
424 | mmhub_v1_8_disable_identity_aperture(adev); |
425 | mmhub_v1_8_setup_vmid_config(adev); |
426 | mmhub_v1_8_program_invalidation(adev); |
427 | |
428 | return 0; |
429 | } |
430 | |
431 | static void mmhub_v1_8_gart_disable(struct amdgpu_device *adev) |
432 | { |
433 | struct amdgpu_vmhub *hub; |
434 | u32 tmp; |
435 | u32 i, j, inst_mask; |
436 | |
437 | /* Disable all tables */ |
438 | inst_mask = adev->aid_mask; |
439 | for_each_inst(j, inst_mask) { |
440 | hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; |
441 | for (i = 0; i < 16; i++) |
442 | WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT0_CNTL, |
443 | i * hub->ctx_distance, 0); |
444 | |
445 | /* Setup TLB control */ |
446 | tmp = RREG32_SOC15(MMHUB, j, regMC_VM_MX_L1_TLB_CNTL); |
447 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, |
448 | 0); |
449 | tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, |
450 | ENABLE_ADVANCED_DRIVER_MODEL, 0); |
451 | WREG32_SOC15(MMHUB, j, regMC_VM_MX_L1_TLB_CNTL, tmp); |
452 | |
453 | if (!amdgpu_sriov_vf(adev)) { |
454 | /* Setup L2 cache */ |
455 | tmp = RREG32_SOC15(MMHUB, j, regVM_L2_CNTL); |
456 | tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, |
457 | 0); |
458 | WREG32_SOC15(MMHUB, j, regVM_L2_CNTL, tmp); |
459 | WREG32_SOC15(MMHUB, j, regVM_L2_CNTL3, 0); |
460 | } |
461 | } |
462 | } |
463 | |
464 | /** |
465 | * mmhub_v1_8_set_fault_enable_default - update GART/VM fault handling |
466 | * |
467 | * @adev: amdgpu_device pointer |
468 | * @value: true redirects VM faults to the default page |
469 | */ |
470 | static void mmhub_v1_8_set_fault_enable_default(struct amdgpu_device *adev, bool value) |
471 | { |
472 | u32 tmp, inst_mask; |
473 | int i; |
474 | |
475 | if (amdgpu_sriov_vf(adev)) |
476 | return; |
477 | |
478 | inst_mask = adev->aid_mask; |
479 | for_each_inst(i, inst_mask) { |
480 | tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL); |
481 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
482 | RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
483 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
484 | PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
485 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
486 | PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
487 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
488 | PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
489 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
490 | TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, |
491 | value); |
492 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
493 | NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
494 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
495 | DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
496 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
497 | VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
498 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
499 | READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
500 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
501 | WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
502 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
503 | EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); |
504 | if (!value) { |
505 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
506 | CRASH_ON_NO_RETRY_FAULT, 1); |
507 | tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, |
508 | CRASH_ON_RETRY_FAULT, 1); |
509 | } |
510 | |
511 | WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL, tmp); |
512 | } |
513 | } |
514 | |
515 | static void mmhub_v1_8_init(struct amdgpu_device *adev) |
516 | { |
517 | struct amdgpu_vmhub *hub; |
518 | u32 inst_mask; |
519 | int i; |
520 | |
521 | inst_mask = adev->aid_mask; |
522 | for_each_inst(i, inst_mask) { |
523 | hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; |
524 | |
525 | hub->ctx0_ptb_addr_lo32 = SOC15_REG_OFFSET(MMHUB, i, |
526 | regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); |
527 | hub->ctx0_ptb_addr_hi32 = SOC15_REG_OFFSET(MMHUB, i, |
528 | regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); |
529 | hub->vm_inv_eng0_req = |
530 | SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_REQ); |
531 | hub->vm_inv_eng0_ack = |
532 | SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_ACK); |
533 | hub->vm_context0_cntl = |
534 | SOC15_REG_OFFSET(MMHUB, i, regVM_CONTEXT0_CNTL); |
535 | hub->vm_l2_pro_fault_status = SOC15_REG_OFFSET(MMHUB, i, |
536 | regVM_L2_PROTECTION_FAULT_STATUS); |
537 | hub->vm_l2_pro_fault_cntl = SOC15_REG_OFFSET(MMHUB, i, |
538 | regVM_L2_PROTECTION_FAULT_CNTL); |
539 | |
540 | hub->ctx_distance = regVM_CONTEXT1_CNTL - regVM_CONTEXT0_CNTL; |
541 | hub->ctx_addr_distance = |
542 | regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - |
543 | regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; |
544 | hub->eng_distance = regVM_INVALIDATE_ENG1_REQ - |
545 | regVM_INVALIDATE_ENG0_REQ; |
546 | hub->eng_addr_distance = regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - |
547 | regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; |
548 | } |
549 | } |
550 | |
551 | static int mmhub_v1_8_set_clockgating(struct amdgpu_device *adev, |
552 | enum amd_clockgating_state state) |
553 | { |
554 | return 0; |
555 | } |
556 | |
557 | static void mmhub_v1_8_get_clockgating(struct amdgpu_device *adev, u64 *flags) |
558 | { |
559 | |
560 | } |
561 | |
562 | const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = { |
563 | .get_fb_location = mmhub_v1_8_get_fb_location, |
564 | .init = mmhub_v1_8_init, |
565 | .gart_enable = mmhub_v1_8_gart_enable, |
566 | .set_fault_enable_default = mmhub_v1_8_set_fault_enable_default, |
567 | .gart_disable = mmhub_v1_8_gart_disable, |
568 | .setup_vm_pt_regs = mmhub_v1_8_setup_vm_pt_regs, |
569 | .set_clockgating = mmhub_v1_8_set_clockgating, |
570 | .get_clockgating = mmhub_v1_8_get_clockgating, |
571 | }; |
572 | |
573 | static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ce_reg_list[] = { |
574 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_CE_ERR_STATUS_LO, regMMEA0_CE_ERR_STATUS_HI), |
575 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0" }, |
576 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_CE_ERR_STATUS_LO, regMMEA1_CE_ERR_STATUS_HI), |
577 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1" }, |
578 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_CE_ERR_STATUS_LO, regMMEA2_CE_ERR_STATUS_HI), |
579 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2" }, |
580 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_CE_ERR_STATUS_LO, regMMEA3_CE_ERR_STATUS_HI), |
581 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3" }, |
582 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_CE_ERR_STATUS_LO, regMMEA4_CE_ERR_STATUS_HI), |
583 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4" }, |
584 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_CE_ERR_STATUS_LO, regMM_CANE_CE_ERR_STATUS_HI), |
585 | 1, 0, "MM_CANE" }, |
586 | }; |
587 | |
588 | static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ue_reg_list[] = { |
589 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_UE_ERR_STATUS_LO, regMMEA0_UE_ERR_STATUS_HI), |
590 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0" }, |
591 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_UE_ERR_STATUS_LO, regMMEA1_UE_ERR_STATUS_HI), |
592 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1" }, |
593 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_UE_ERR_STATUS_LO, regMMEA2_UE_ERR_STATUS_HI), |
594 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2" }, |
595 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_UE_ERR_STATUS_LO, regMMEA3_UE_ERR_STATUS_HI), |
596 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3" }, |
597 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_UE_ERR_STATUS_LO, regMMEA4_UE_ERR_STATUS_HI), |
598 | 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4" }, |
599 | {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_UE_ERR_STATUS_LO, regMM_CANE_UE_ERR_STATUS_HI), |
600 | 1, 0, "MM_CANE" }, |
601 | }; |
602 | |
603 | static const struct amdgpu_ras_memory_id_entry mmhub_v1_8_ras_memory_list[] = { |
604 | {AMDGPU_MMHUB_WGMI_PAGEMEM, "MMEA_WGMI_PAGEMEM" }, |
605 | {AMDGPU_MMHUB_RGMI_PAGEMEM, "MMEA_RGMI_PAGEMEM" }, |
606 | {AMDGPU_MMHUB_WDRAM_PAGEMEM, "MMEA_WDRAM_PAGEMEM" }, |
607 | {AMDGPU_MMHUB_RDRAM_PAGEMEM, "MMEA_RDRAM_PAGEMEM" }, |
608 | {AMDGPU_MMHUB_WIO_CMDMEM, "MMEA_WIO_CMDMEM" }, |
609 | {AMDGPU_MMHUB_RIO_CMDMEM, "MMEA_RIO_CMDMEM" }, |
610 | {AMDGPU_MMHUB_WGMI_CMDMEM, "MMEA_WGMI_CMDMEM" }, |
611 | {AMDGPU_MMHUB_RGMI_CMDMEM, "MMEA_RGMI_CMDMEM" }, |
612 | {AMDGPU_MMHUB_WDRAM_CMDMEM, "MMEA_WDRAM_CMDMEM" }, |
613 | {AMDGPU_MMHUB_RDRAM_CMDMEM, "MMEA_RDRAM_CMDMEM" }, |
614 | {AMDGPU_MMHUB_MAM_DMEM0, "MMEA_MAM_DMEM0" }, |
615 | {AMDGPU_MMHUB_MAM_DMEM1, "MMEA_MAM_DMEM1" }, |
616 | {AMDGPU_MMHUB_MAM_DMEM2, "MMEA_MAM_DMEM2" }, |
617 | {AMDGPU_MMHUB_MAM_DMEM3, "MMEA_MAM_DMEM3" }, |
618 | {AMDGPU_MMHUB_WRET_TAGMEM, "MMEA_WRET_TAGMEM" }, |
619 | {AMDGPU_MMHUB_RRET_TAGMEM, "MMEA_RRET_TAGMEM" }, |
620 | {AMDGPU_MMHUB_WIO_DATAMEM, "MMEA_WIO_DATAMEM" }, |
621 | {AMDGPU_MMHUB_WGMI_DATAMEM, "MMEA_WGMI_DATAMEM" }, |
622 | {AMDGPU_MMHUB_WDRAM_DATAMEM, "MMEA_WDRAM_DATAMEM" }, |
623 | }; |
624 | |
625 | static void mmhub_v1_8_inst_query_ras_error_count(struct amdgpu_device *adev, |
626 | uint32_t mmhub_inst, |
627 | void *ras_err_status) |
628 | { |
629 | struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; |
630 | unsigned long ue_count = 0, ce_count = 0; |
631 | |
632 | /* NOTE: mmhub is converted by aid_mask and the range is 0-3, |
633 | * which can be used as die ID directly */ |
634 | struct amdgpu_smuio_mcm_config_info mcm_info = { |
635 | .socket_id = adev->smuio.funcs->get_socket_id(adev), |
636 | .die_id = mmhub_inst, |
637 | }; |
638 | |
639 | amdgpu_ras_inst_query_ras_error_count(adev, |
640 | reg_list: mmhub_v1_8_ce_reg_list, |
641 | ARRAY_SIZE(mmhub_v1_8_ce_reg_list), |
642 | mem_list: mmhub_v1_8_ras_memory_list, |
643 | ARRAY_SIZE(mmhub_v1_8_ras_memory_list), |
644 | instance: mmhub_inst, |
645 | err_type: AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE, |
646 | err_count: &ce_count); |
647 | amdgpu_ras_inst_query_ras_error_count(adev, |
648 | reg_list: mmhub_v1_8_ue_reg_list, |
649 | ARRAY_SIZE(mmhub_v1_8_ue_reg_list), |
650 | mem_list: mmhub_v1_8_ras_memory_list, |
651 | ARRAY_SIZE(mmhub_v1_8_ras_memory_list), |
652 | instance: mmhub_inst, |
653 | err_type: AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, |
654 | err_count: &ue_count); |
655 | |
656 | amdgpu_ras_error_statistic_ce_count(err_data, mcm_info: &mcm_info, NULL, count: ce_count); |
657 | amdgpu_ras_error_statistic_ue_count(err_data, mcm_info: &mcm_info, NULL, count: ue_count); |
658 | } |
659 | |
660 | static void mmhub_v1_8_query_ras_error_count(struct amdgpu_device *adev, |
661 | void *ras_err_status) |
662 | { |
663 | uint32_t inst_mask; |
664 | uint32_t i; |
665 | |
666 | if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__MMHUB)) { |
667 | dev_warn(adev->dev, "MMHUB RAS is not supported\n" ); |
668 | return; |
669 | } |
670 | |
671 | inst_mask = adev->aid_mask; |
672 | for_each_inst(i, inst_mask) |
673 | mmhub_v1_8_inst_query_ras_error_count(adev, mmhub_inst: i, ras_err_status); |
674 | } |
675 | |
676 | static void mmhub_v1_8_inst_reset_ras_error_count(struct amdgpu_device *adev, |
677 | uint32_t mmhub_inst) |
678 | { |
679 | amdgpu_ras_inst_reset_ras_error_count(adev, |
680 | reg_list: mmhub_v1_8_ce_reg_list, |
681 | ARRAY_SIZE(mmhub_v1_8_ce_reg_list), |
682 | instance: mmhub_inst); |
683 | amdgpu_ras_inst_reset_ras_error_count(adev, |
684 | reg_list: mmhub_v1_8_ue_reg_list, |
685 | ARRAY_SIZE(mmhub_v1_8_ue_reg_list), |
686 | instance: mmhub_inst); |
687 | } |
688 | |
689 | static void mmhub_v1_8_reset_ras_error_count(struct amdgpu_device *adev) |
690 | { |
691 | uint32_t inst_mask; |
692 | uint32_t i; |
693 | |
694 | if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__MMHUB)) { |
695 | dev_warn(adev->dev, "MMHUB RAS is not supported\n" ); |
696 | return; |
697 | } |
698 | |
699 | inst_mask = adev->aid_mask; |
700 | for_each_inst(i, inst_mask) |
701 | mmhub_v1_8_inst_reset_ras_error_count(adev, mmhub_inst: i); |
702 | } |
703 | |
704 | static const struct amdgpu_ras_block_hw_ops mmhub_v1_8_ras_hw_ops = { |
705 | .query_ras_error_count = mmhub_v1_8_query_ras_error_count, |
706 | .reset_ras_error_count = mmhub_v1_8_reset_ras_error_count, |
707 | }; |
708 | |
709 | static int mmhub_v1_8_aca_bank_generate_report(struct aca_handle *handle, |
710 | struct aca_bank *bank, enum aca_error_type type, |
711 | struct aca_bank_report *report, void *data) |
712 | { |
713 | u64 status, misc0; |
714 | int ret; |
715 | |
716 | status = bank->regs[ACA_REG_IDX_STATUS]; |
717 | if ((type == ACA_ERROR_TYPE_UE && |
718 | ACA_REG__STATUS__ERRORCODEEXT(status) == ACA_EXTERROR_CODE_FAULT) || |
719 | (type == ACA_ERROR_TYPE_CE && |
720 | ACA_REG__STATUS__ERRORCODEEXT(status) == ACA_EXTERROR_CODE_CE)) { |
721 | |
722 | ret = aca_bank_info_decode(bank, info: &report->info); |
723 | if (ret) |
724 | return ret; |
725 | |
726 | misc0 = bank->regs[ACA_REG_IDX_MISC0]; |
727 | report->count[type] = ACA_REG__MISC0__ERRCNT(misc0); |
728 | } |
729 | |
730 | return 0; |
731 | } |
732 | |
733 | /* reference to smu driver if header file */ |
734 | static int mmhub_v1_8_err_codes[] = { |
735 | 0, 1, 2, 3, 4, /* CODE_DAGB0 - 4 */ |
736 | 5, 6, 7, 8, 9, /* CODE_EA0 - 4 */ |
737 | 10, /* CODE_UTCL2_ROUTER */ |
738 | 11, /* CODE_VML2 */ |
739 | 12, /* CODE_VML2_WALKER */ |
740 | 13, /* CODE_MMCANE */ |
741 | }; |
742 | |
743 | static bool mmhub_v1_8_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, |
744 | enum aca_error_type type, void *data) |
745 | { |
746 | u32 instlo; |
747 | |
748 | instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); |
749 | instlo &= GENMASK(31, 1); |
750 | |
751 | if (instlo != mmSMNAID_AID0_MCA_SMU) |
752 | return false; |
753 | |
754 | if (aca_bank_check_error_codes(adev: handle->adev, bank, |
755 | err_codes: mmhub_v1_8_err_codes, |
756 | ARRAY_SIZE(mmhub_v1_8_err_codes))) |
757 | return false; |
758 | |
759 | return true; |
760 | } |
761 | |
762 | static const struct aca_bank_ops mmhub_v1_8_aca_bank_ops = { |
763 | .aca_bank_generate_report = mmhub_v1_8_aca_bank_generate_report, |
764 | .aca_bank_is_valid = mmhub_v1_8_aca_bank_is_valid, |
765 | }; |
766 | |
767 | static const struct aca_info mmhub_v1_8_aca_info = { |
768 | .hwip = ACA_HWIP_TYPE_SMU, |
769 | .mask = ACA_ERROR_UE_MASK, |
770 | .bank_ops = &mmhub_v1_8_aca_bank_ops, |
771 | }; |
772 | |
773 | static int mmhub_v1_8_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) |
774 | { |
775 | int r; |
776 | |
777 | r = amdgpu_ras_block_late_init(adev, ras_block); |
778 | if (r) |
779 | return r; |
780 | |
781 | r = amdgpu_ras_bind_aca(adev, blk: AMDGPU_RAS_BLOCK__MMHUB, |
782 | aca_info: &mmhub_v1_8_aca_info, NULL); |
783 | if (r) |
784 | goto late_fini; |
785 | |
786 | return 0; |
787 | |
788 | late_fini: |
789 | amdgpu_ras_block_late_fini(adev, ras_block); |
790 | |
791 | return r; |
792 | } |
793 | |
794 | struct amdgpu_mmhub_ras mmhub_v1_8_ras = { |
795 | .ras_block = { |
796 | .hw_ops = &mmhub_v1_8_ras_hw_ops, |
797 | .ras_late_init = mmhub_v1_8_ras_late_init, |
798 | }, |
799 | }; |
800 | |