1 | /* |
2 | * Copyright 2021 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | */ |
23 | #include "amdgpu.h" |
24 | #include "amdgpu_atombios.h" |
25 | #include "nbio_v7_7.h" |
26 | |
27 | #include "nbio/nbio_7_7_0_offset.h" |
28 | #include "nbio/nbio_7_7_0_sh_mask.h" |
29 | #include <uapi/linux/kfd_ioctl.h> |
30 | |
31 | static void nbio_v7_7_remap_hdp_registers(struct amdgpu_device *adev) |
32 | { |
33 | WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL, |
34 | adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); |
35 | WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL, |
36 | adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); |
37 | } |
38 | |
39 | static u32 nbio_v7_7_get_rev_id(struct amdgpu_device *adev) |
40 | { |
41 | u32 tmp; |
42 | |
43 | tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0); |
44 | tmp &= RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK; |
45 | tmp >>= RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT; |
46 | |
47 | return tmp; |
48 | } |
49 | |
50 | static void nbio_v7_7_mc_access_enable(struct amdgpu_device *adev, bool enable) |
51 | { |
52 | if (enable) |
53 | WREG32_SOC15(NBIO, 0, regBIF_BX1_BIF_FB_EN, |
54 | BIF_BX1_BIF_FB_EN__FB_READ_EN_MASK | |
55 | BIF_BX1_BIF_FB_EN__FB_WRITE_EN_MASK); |
56 | else |
57 | WREG32_SOC15(NBIO, 0, regBIF_BX1_BIF_FB_EN, 0); |
58 | } |
59 | |
60 | static u32 nbio_v7_7_get_memsize(struct amdgpu_device *adev) |
61 | { |
62 | return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE); |
63 | } |
64 | |
65 | static void nbio_v7_7_sdma_doorbell_range(struct amdgpu_device *adev, int instance, |
66 | bool use_doorbell, int doorbell_index, |
67 | int doorbell_size) |
68 | { |
69 | u32 reg = SOC15_REG_OFFSET(NBIO, 0, regGDC0_BIF_CSDMA_DOORBELL_RANGE); |
70 | u32 doorbell_range = RREG32_PCIE_PORT(reg); |
71 | |
72 | if (use_doorbell) { |
73 | doorbell_range = REG_SET_FIELD(doorbell_range, |
74 | GDC0_BIF_CSDMA_DOORBELL_RANGE, |
75 | OFFSET, doorbell_index); |
76 | doorbell_range = REG_SET_FIELD(doorbell_range, |
77 | GDC0_BIF_CSDMA_DOORBELL_RANGE, |
78 | SIZE, doorbell_size); |
79 | } else { |
80 | doorbell_range = REG_SET_FIELD(doorbell_range, |
81 | GDC0_BIF_SDMA0_DOORBELL_RANGE, |
82 | SIZE, 0); |
83 | } |
84 | |
85 | WREG32_PCIE_PORT(reg, doorbell_range); |
86 | } |
87 | |
88 | static void nbio_v7_7_vcn_doorbell_range(struct amdgpu_device *adev, bool use_doorbell, |
89 | int doorbell_index, int instance) |
90 | { |
91 | u32 reg = SOC15_REG_OFFSET(NBIO, 0, regGDC0_BIF_VCN0_DOORBELL_RANGE); |
92 | u32 doorbell_range = RREG32_PCIE_PORT(reg); |
93 | |
94 | if (use_doorbell) { |
95 | doorbell_range = REG_SET_FIELD(doorbell_range, |
96 | GDC0_BIF_VCN0_DOORBELL_RANGE, OFFSET, |
97 | doorbell_index); |
98 | doorbell_range = REG_SET_FIELD(doorbell_range, |
99 | GDC0_BIF_VCN0_DOORBELL_RANGE, SIZE, 8); |
100 | } else { |
101 | doorbell_range = REG_SET_FIELD(doorbell_range, |
102 | GDC0_BIF_VCN0_DOORBELL_RANGE, SIZE, 0); |
103 | } |
104 | |
105 | WREG32_PCIE_PORT(reg, doorbell_range); |
106 | } |
107 | |
108 | static void nbio_v7_7_enable_doorbell_aperture(struct amdgpu_device *adev, |
109 | bool enable) |
110 | { |
111 | u32 reg; |
112 | |
113 | reg = RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_0_RCC_DOORBELL_APER_EN); |
114 | reg = REG_SET_FIELD(reg, RCC_DEV0_EPF0_0_RCC_DOORBELL_APER_EN, |
115 | BIF_DOORBELL_APER_EN, enable ? 1 : 0); |
116 | |
117 | WREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_0_RCC_DOORBELL_APER_EN, reg); |
118 | } |
119 | |
120 | static void nbio_v7_7_enable_doorbell_selfring_aperture(struct amdgpu_device *adev, |
121 | bool enable) |
122 | { |
123 | u32 tmp = 0; |
124 | |
125 | if (enable) { |
126 | tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, |
127 | DOORBELL_SELFRING_GPA_APER_EN, 1) | |
128 | REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, |
129 | DOORBELL_SELFRING_GPA_APER_MODE, 1) | |
130 | REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, |
131 | DOORBELL_SELFRING_GPA_APER_SIZE, 0); |
132 | |
133 | WREG32_SOC15(NBIO, 0, |
134 | regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW, |
135 | lower_32_bits(adev->doorbell.base)); |
136 | WREG32_SOC15(NBIO, 0, |
137 | regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH, |
138 | upper_32_bits(adev->doorbell.base)); |
139 | } |
140 | |
141 | WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, |
142 | tmp); |
143 | } |
144 | |
145 | |
146 | static void nbio_v7_7_ih_doorbell_range(struct amdgpu_device *adev, |
147 | bool use_doorbell, int doorbell_index) |
148 | { |
149 | u32 ih_doorbell_range = RREG32_SOC15(NBIO, 0, |
150 | regGDC0_BIF_IH_DOORBELL_RANGE); |
151 | |
152 | if (use_doorbell) { |
153 | ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, |
154 | GDC0_BIF_IH_DOORBELL_RANGE, OFFSET, |
155 | doorbell_index); |
156 | ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, |
157 | GDC0_BIF_IH_DOORBELL_RANGE, SIZE, |
158 | 2); |
159 | } else { |
160 | ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, |
161 | GDC0_BIF_IH_DOORBELL_RANGE, SIZE, |
162 | 0); |
163 | } |
164 | |
165 | WREG32_SOC15(NBIO, 0, regGDC0_BIF_IH_DOORBELL_RANGE, |
166 | ih_doorbell_range); |
167 | } |
168 | |
169 | static void nbio_v7_7_ih_control(struct amdgpu_device *adev) |
170 | { |
171 | u32 interrupt_cntl; |
172 | |
173 | /* setup interrupt control */ |
174 | WREG32_SOC15(NBIO, 0, regBIF_BX1_INTERRUPT_CNTL2, |
175 | adev->dummy_page_addr >> 8); |
176 | |
177 | interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX1_INTERRUPT_CNTL); |
178 | /* |
179 | * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi |
180 | * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN |
181 | */ |
182 | interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX1_INTERRUPT_CNTL, |
183 | IH_DUMMY_RD_OVERRIDE, 0); |
184 | |
185 | /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */ |
186 | interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX1_INTERRUPT_CNTL, |
187 | IH_REQ_NONSNOOP_EN, 0); |
188 | |
189 | WREG32_SOC15(NBIO, 0, regBIF_BX1_INTERRUPT_CNTL, interrupt_cntl); |
190 | } |
191 | |
192 | static u32 nbio_v7_7_get_hdp_flush_req_offset(struct amdgpu_device *adev) |
193 | { |
194 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_REQ); |
195 | } |
196 | |
197 | static u32 nbio_v7_7_get_hdp_flush_done_offset(struct amdgpu_device *adev) |
198 | { |
199 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_DONE); |
200 | } |
201 | |
202 | static u32 nbio_v7_7_get_pcie_index_offset(struct amdgpu_device *adev) |
203 | { |
204 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2); |
205 | } |
206 | |
207 | static u32 nbio_v7_7_get_pcie_data_offset(struct amdgpu_device *adev) |
208 | { |
209 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA2); |
210 | } |
211 | |
212 | static u32 nbio_v7_7_get_pcie_port_index_offset(struct amdgpu_device *adev) |
213 | { |
214 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_INDEX); |
215 | } |
216 | |
217 | static u32 nbio_v7_7_get_pcie_port_data_offset(struct amdgpu_device *adev) |
218 | { |
219 | return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_DATA); |
220 | } |
221 | |
222 | const struct nbio_hdp_flush_reg nbio_v7_7_hdp_flush_reg = { |
223 | .ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK, |
224 | .ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK, |
225 | .ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK, |
226 | .ref_and_mask_cp3 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK, |
227 | .ref_and_mask_cp4 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK, |
228 | .ref_and_mask_cp5 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK, |
229 | .ref_and_mask_cp6 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK, |
230 | .ref_and_mask_cp7 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK, |
231 | .ref_and_mask_cp8 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK, |
232 | .ref_and_mask_cp9 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK, |
233 | .ref_and_mask_sdma0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK, |
234 | .ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK, |
235 | }; |
236 | |
237 | static void nbio_v7_7_init_registers(struct amdgpu_device *adev) |
238 | { |
239 | uint32_t def, data; |
240 | |
241 | def = data = RREG32_SOC15(NBIO, 0, regBIF0_PCIE_MST_CTRL_3); |
242 | data = REG_SET_FIELD(data, BIF0_PCIE_MST_CTRL_3, |
243 | CI_SWUS_MAX_READ_REQUEST_SIZE_MODE, 1); |
244 | data = REG_SET_FIELD(data, BIF0_PCIE_MST_CTRL_3, |
245 | CI_SWUS_MAX_READ_REQUEST_SIZE_PRIV, 1); |
246 | |
247 | if (def != data) |
248 | WREG32_SOC15(NBIO, 0, regBIF0_PCIE_MST_CTRL_3, data); |
249 | |
250 | } |
251 | |
252 | static void nbio_v7_7_update_medium_grain_clock_gating(struct amdgpu_device *adev, |
253 | bool enable) |
254 | { |
255 | uint32_t def, data; |
256 | |
257 | if (!(adev->cg_flags & AMD_CG_SUPPORT_BIF_MGCG)) |
258 | return; |
259 | |
260 | def = data = RREG32_SOC15(NBIO, 0, regBIF0_CPM_CONTROL); |
261 | if (enable) { |
262 | data |= (BIF0_CPM_CONTROL__LCLK_DYN_GATE_ENABLE_MASK | |
263 | BIF0_CPM_CONTROL__TXCLK_DYN_GATE_ENABLE_MASK | |
264 | BIF0_CPM_CONTROL__TXCLK_LCNT_GATE_ENABLE_MASK | |
265 | BIF0_CPM_CONTROL__TXCLK_REGS_GATE_ENABLE_MASK | |
266 | BIF0_CPM_CONTROL__TXCLK_PRBS_GATE_ENABLE_MASK | |
267 | BIF0_CPM_CONTROL__REFCLK_REGS_GATE_ENABLE_MASK); |
268 | } else { |
269 | data &= ~(BIF0_CPM_CONTROL__LCLK_DYN_GATE_ENABLE_MASK | |
270 | BIF0_CPM_CONTROL__TXCLK_DYN_GATE_ENABLE_MASK | |
271 | BIF0_CPM_CONTROL__TXCLK_LCNT_GATE_ENABLE_MASK | |
272 | BIF0_CPM_CONTROL__TXCLK_REGS_GATE_ENABLE_MASK | |
273 | BIF0_CPM_CONTROL__TXCLK_PRBS_GATE_ENABLE_MASK | |
274 | BIF0_CPM_CONTROL__REFCLK_REGS_GATE_ENABLE_MASK); |
275 | } |
276 | |
277 | if (def != data) |
278 | WREG32_SOC15(NBIO, 0, regBIF0_CPM_CONTROL, data); |
279 | } |
280 | |
281 | static void nbio_v7_7_update_medium_grain_light_sleep(struct amdgpu_device *adev, |
282 | bool enable) |
283 | { |
284 | uint32_t def, data; |
285 | |
286 | if (!(adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) |
287 | return; |
288 | |
289 | def = data = RREG32_SOC15(NBIO, 0, regBIF0_PCIE_CNTL2); |
290 | if (enable) |
291 | data |= BIF0_PCIE_CNTL2__SLV_MEM_LS_EN_MASK; |
292 | else |
293 | data &= ~BIF0_PCIE_CNTL2__SLV_MEM_LS_EN_MASK; |
294 | |
295 | if (def != data) |
296 | WREG32_SOC15(NBIO, 0, regBIF0_PCIE_CNTL2, data); |
297 | |
298 | def = data = RREG32_SOC15(NBIO, 0, regBIF0_PCIE_TX_POWER_CTRL_1); |
299 | if (enable) { |
300 | data |= (BIF0_PCIE_TX_POWER_CTRL_1__MST_MEM_LS_EN_MASK | |
301 | BIF0_PCIE_TX_POWER_CTRL_1__REPLAY_MEM_LS_EN_MASK); |
302 | } else { |
303 | data &= ~(BIF0_PCIE_TX_POWER_CTRL_1__MST_MEM_LS_EN_MASK | |
304 | BIF0_PCIE_TX_POWER_CTRL_1__REPLAY_MEM_LS_EN_MASK); |
305 | } |
306 | |
307 | if (def != data) |
308 | WREG32_SOC15(NBIO, 0, regBIF0_PCIE_TX_POWER_CTRL_1, data); |
309 | } |
310 | |
311 | static void nbio_v7_7_get_clockgating_state(struct amdgpu_device *adev, |
312 | u64 *flags) |
313 | { |
314 | uint32_t data; |
315 | |
316 | /* AMD_CG_SUPPORT_BIF_MGCG */ |
317 | data = RREG32_SOC15(NBIO, 0, regBIF0_CPM_CONTROL); |
318 | if (data & BIF0_CPM_CONTROL__LCLK_DYN_GATE_ENABLE_MASK) |
319 | *flags |= AMD_CG_SUPPORT_BIF_MGCG; |
320 | |
321 | /* AMD_CG_SUPPORT_BIF_LS */ |
322 | data = RREG32_SOC15(NBIO, 0, regBIF0_PCIE_CNTL2); |
323 | if (data & BIF0_PCIE_CNTL2__SLV_MEM_LS_EN_MASK) |
324 | *flags |= AMD_CG_SUPPORT_BIF_LS; |
325 | } |
326 | |
327 | const struct amdgpu_nbio_funcs nbio_v7_7_funcs = { |
328 | .get_hdp_flush_req_offset = nbio_v7_7_get_hdp_flush_req_offset, |
329 | .get_hdp_flush_done_offset = nbio_v7_7_get_hdp_flush_done_offset, |
330 | .get_pcie_index_offset = nbio_v7_7_get_pcie_index_offset, |
331 | .get_pcie_data_offset = nbio_v7_7_get_pcie_data_offset, |
332 | .get_pcie_port_index_offset = nbio_v7_7_get_pcie_port_index_offset, |
333 | .get_pcie_port_data_offset = nbio_v7_7_get_pcie_port_data_offset, |
334 | .get_rev_id = nbio_v7_7_get_rev_id, |
335 | .mc_access_enable = nbio_v7_7_mc_access_enable, |
336 | .get_memsize = nbio_v7_7_get_memsize, |
337 | .sdma_doorbell_range = nbio_v7_7_sdma_doorbell_range, |
338 | .vcn_doorbell_range = nbio_v7_7_vcn_doorbell_range, |
339 | .enable_doorbell_aperture = nbio_v7_7_enable_doorbell_aperture, |
340 | .enable_doorbell_selfring_aperture = nbio_v7_7_enable_doorbell_selfring_aperture, |
341 | .ih_doorbell_range = nbio_v7_7_ih_doorbell_range, |
342 | .update_medium_grain_clock_gating = nbio_v7_7_update_medium_grain_clock_gating, |
343 | .update_medium_grain_light_sleep = nbio_v7_7_update_medium_grain_light_sleep, |
344 | .get_clockgating_state = nbio_v7_7_get_clockgating_state, |
345 | .ih_control = nbio_v7_7_ih_control, |
346 | .init_registers = nbio_v7_7_init_registers, |
347 | .remap_hdp_registers = nbio_v7_7_remap_hdp_registers, |
348 | }; |
349 | |