1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2020-2023 Intel Corporation |
4 | */ |
5 | |
6 | #include "ivpu_drv.h" |
7 | #include "ivpu_fw.h" |
8 | #include "ivpu_hw.h" |
9 | #include "ivpu_hw_40xx_reg.h" |
10 | #include "ivpu_hw_reg_io.h" |
11 | #include "ivpu_ipc.h" |
12 | #include "ivpu_mmu.h" |
13 | #include "ivpu_pm.h" |
14 | |
15 | #include <linux/dmi.h> |
16 | |
17 | #define TILE_MAX_NUM 6 |
18 | #define TILE_MAX_MASK 0x3f |
19 | |
20 | #define LNL_HW_ID 0x4040 |
21 | |
22 | #define SKU_TILE_SHIFT 0u |
23 | #define SKU_TILE_MASK 0x0000ffffu |
24 | #define SKU_HW_ID_SHIFT 16u |
25 | #define SKU_HW_ID_MASK 0xffff0000u |
26 | |
27 | #define PLL_CONFIG_DEFAULT 0x0 |
28 | #define PLL_CDYN_DEFAULT 0x80 |
29 | #define PLL_EPP_DEFAULT 0x80 |
30 | #define PLL_REF_CLK_FREQ (50 * 1000000) |
31 | #define PLL_RATIO_TO_FREQ(x) ((x) * PLL_REF_CLK_FREQ) |
32 | |
33 | #define PLL_PROFILING_FREQ_DEFAULT 38400000 |
34 | #define PLL_PROFILING_FREQ_HIGH 400000000 |
35 | |
36 | #define TIM_SAFE_ENABLE 0xf1d0dead |
37 | #define TIM_WATCHDOG_RESET_VALUE 0xffffffff |
38 | |
39 | #define TIMEOUT_US (150 * USEC_PER_MSEC) |
40 | #define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC) |
41 | #define PLL_TIMEOUT_US (1500 * USEC_PER_MSEC) |
42 | #define IDLE_TIMEOUT_US (5 * USEC_PER_MSEC) |
43 | |
44 | #define WEIGHTS_DEFAULT 0xf711f711u |
45 | #define WEIGHTS_ATS_DEFAULT 0x0000f711u |
46 | |
47 | #define ICB_0_IRQ_MASK ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \ |
48 | (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \ |
49 | (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \ |
50 | (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \ |
51 | (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \ |
52 | (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \ |
53 | (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT))) |
54 | |
55 | #define ICB_1_IRQ_MASK ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \ |
56 | (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \ |
57 | (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT))) |
58 | |
59 | #define ICB_0_1_IRQ_MASK ((((u64)ICB_1_IRQ_MASK) << 32) | ICB_0_IRQ_MASK) |
60 | |
61 | #define BUTTRESS_IRQ_MASK ((REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR)) | \ |
62 | (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR)) | \ |
63 | (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR)) | \ |
64 | (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR)) | \ |
65 | (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR)) | \ |
66 | (REG_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR))) |
67 | |
68 | #define BUTTRESS_IRQ_ENABLE_MASK ((u32)~BUTTRESS_IRQ_MASK) |
69 | #define BUTTRESS_IRQ_DISABLE_MASK ((u32)-1) |
70 | |
71 | #define ITF_FIREWALL_VIOLATION_MASK ((REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \ |
72 | (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \ |
73 | (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \ |
74 | (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \ |
75 | (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \ |
76 | (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \ |
77 | (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX))) |
78 | |
79 | static char *ivpu_platform_to_str(u32 platform) |
80 | { |
81 | switch (platform) { |
82 | case IVPU_PLATFORM_SILICON: |
83 | return "SILICON" ; |
84 | case IVPU_PLATFORM_SIMICS: |
85 | return "SIMICS" ; |
86 | case IVPU_PLATFORM_FPGA: |
87 | return "FPGA" ; |
88 | default: |
89 | return "Invalid platform" ; |
90 | } |
91 | } |
92 | |
93 | static const struct dmi_system_id ivpu_dmi_platform_simulation[] = { |
94 | { |
95 | .ident = "Intel Simics" , |
96 | .matches = { |
97 | DMI_MATCH(DMI_BOARD_NAME, "lnlrvp" ), |
98 | DMI_MATCH(DMI_BOARD_VERSION, "1.0" ), |
99 | DMI_MATCH(DMI_BOARD_SERIAL, "123456789" ), |
100 | }, |
101 | }, |
102 | { |
103 | .ident = "Intel Simics" , |
104 | .matches = { |
105 | DMI_MATCH(DMI_BOARD_NAME, "Simics" ), |
106 | }, |
107 | }, |
108 | { } |
109 | }; |
110 | |
111 | static void ivpu_hw_read_platform(struct ivpu_device *vdev) |
112 | { |
113 | if (dmi_check_system(list: ivpu_dmi_platform_simulation)) |
114 | vdev->platform = IVPU_PLATFORM_SIMICS; |
115 | else |
116 | vdev->platform = IVPU_PLATFORM_SILICON; |
117 | |
118 | ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n" , |
119 | ivpu_platform_to_str(vdev->platform), vdev->platform); |
120 | } |
121 | |
122 | static void ivpu_hw_wa_init(struct ivpu_device *vdev) |
123 | { |
124 | vdev->wa.punit_disabled = ivpu_is_fpga(vdev); |
125 | vdev->wa.clear_runtime_mem = false; |
126 | |
127 | if (ivpu_hw_gen(vdev) == IVPU_HW_40XX) |
128 | vdev->wa.disable_clock_relinquish = true; |
129 | |
130 | IVPU_PRINT_WA(punit_disabled); |
131 | IVPU_PRINT_WA(clear_runtime_mem); |
132 | IVPU_PRINT_WA(disable_clock_relinquish); |
133 | } |
134 | |
135 | static void ivpu_hw_timeouts_init(struct ivpu_device *vdev) |
136 | { |
137 | if (ivpu_is_fpga(vdev)) { |
138 | vdev->timeout.boot = 100000; |
139 | vdev->timeout.jsm = 50000; |
140 | vdev->timeout.tdr = 2000000; |
141 | vdev->timeout.reschedule_suspend = 1000; |
142 | vdev->timeout.autosuspend = -1; |
143 | vdev->timeout.d0i3_entry_msg = 500; |
144 | } else if (ivpu_is_simics(vdev)) { |
145 | vdev->timeout.boot = 50; |
146 | vdev->timeout.jsm = 500; |
147 | vdev->timeout.tdr = 10000; |
148 | vdev->timeout.reschedule_suspend = 10; |
149 | vdev->timeout.autosuspend = -1; |
150 | vdev->timeout.d0i3_entry_msg = 100; |
151 | } else { |
152 | vdev->timeout.boot = 1000; |
153 | vdev->timeout.jsm = 500; |
154 | vdev->timeout.tdr = 2000; |
155 | vdev->timeout.reschedule_suspend = 10; |
156 | vdev->timeout.autosuspend = 10; |
157 | vdev->timeout.d0i3_entry_msg = 5; |
158 | } |
159 | } |
160 | |
161 | static int ivpu_pll_wait_for_cmd_send(struct ivpu_device *vdev) |
162 | { |
163 | return REGB_POLL_FLD(VPU_40XX_BUTTRESS_WP_REQ_CMD, SEND, 0, PLL_TIMEOUT_US); |
164 | } |
165 | |
166 | static int ivpu_pll_cmd_send(struct ivpu_device *vdev, u16 min_ratio, u16 max_ratio, |
167 | u16 target_ratio, u16 epp, u16 config, u16 cdyn) |
168 | { |
169 | int ret; |
170 | u32 val; |
171 | |
172 | ret = ivpu_pll_wait_for_cmd_send(vdev); |
173 | if (ret) { |
174 | ivpu_err(vdev, "Failed to sync before WP request: %d\n" , ret); |
175 | return ret; |
176 | } |
177 | |
178 | val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0); |
179 | val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, MIN_RATIO, min_ratio, val); |
180 | val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, MAX_RATIO, max_ratio, val); |
181 | REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD0, val); |
182 | |
183 | val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1); |
184 | val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, TARGET_RATIO, target_ratio, val); |
185 | val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, EPP, epp, val); |
186 | REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD1, val); |
187 | |
188 | val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2); |
189 | val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, CONFIG, config, val); |
190 | val = REG_SET_FLD_NUM(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, CDYN, cdyn, val); |
191 | REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_PAYLOAD2, val); |
192 | |
193 | val = REGB_RD32(VPU_40XX_BUTTRESS_WP_REQ_CMD); |
194 | val = REG_SET_FLD(VPU_40XX_BUTTRESS_WP_REQ_CMD, SEND, val); |
195 | REGB_WR32(VPU_40XX_BUTTRESS_WP_REQ_CMD, val); |
196 | |
197 | ret = ivpu_pll_wait_for_cmd_send(vdev); |
198 | if (ret) |
199 | ivpu_err(vdev, "Failed to sync after WP request: %d\n" , ret); |
200 | |
201 | return ret; |
202 | } |
203 | |
204 | static int ivpu_pll_wait_for_status_ready(struct ivpu_device *vdev) |
205 | { |
206 | return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, READY, 1, PLL_TIMEOUT_US); |
207 | } |
208 | |
209 | static int ivpu_wait_for_clock_own_resource_ack(struct ivpu_device *vdev) |
210 | { |
211 | if (ivpu_is_simics(vdev)) |
212 | return 0; |
213 | |
214 | return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, CLOCK_RESOURCE_OWN_ACK, 1, TIMEOUT_US); |
215 | } |
216 | |
217 | static void ivpu_pll_init_frequency_ratios(struct ivpu_device *vdev) |
218 | { |
219 | struct ivpu_hw_info *hw = vdev->hw; |
220 | u8 fuse_min_ratio, fuse_pn_ratio, fuse_max_ratio; |
221 | u32 fmin_fuse, fmax_fuse; |
222 | |
223 | fmin_fuse = REGB_RD32(VPU_40XX_BUTTRESS_FMIN_FUSE); |
224 | fuse_min_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMIN_FUSE, MIN_RATIO, fmin_fuse); |
225 | fuse_pn_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMIN_FUSE, PN_RATIO, fmin_fuse); |
226 | |
227 | fmax_fuse = REGB_RD32(VPU_40XX_BUTTRESS_FMAX_FUSE); |
228 | fuse_max_ratio = REG_GET_FLD(VPU_40XX_BUTTRESS_FMAX_FUSE, MAX_RATIO, fmax_fuse); |
229 | |
230 | hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, fuse_min_ratio, fuse_max_ratio); |
231 | hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, fuse_max_ratio); |
232 | hw->pll.pn_ratio = clamp_t(u8, fuse_pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio); |
233 | } |
234 | |
235 | static int ivpu_pll_drive(struct ivpu_device *vdev, bool enable) |
236 | { |
237 | u16 config = enable ? PLL_CONFIG_DEFAULT : 0; |
238 | u16 cdyn = enable ? PLL_CDYN_DEFAULT : 0; |
239 | u16 epp = enable ? PLL_EPP_DEFAULT : 0; |
240 | struct ivpu_hw_info *hw = vdev->hw; |
241 | u16 target_ratio = hw->pll.pn_ratio; |
242 | int ret; |
243 | |
244 | ivpu_dbg(vdev, PM, "PLL workpoint request: %u Hz, epp: 0x%x, config: 0x%x, cdyn: 0x%x\n" , |
245 | PLL_RATIO_TO_FREQ(target_ratio), epp, config, cdyn); |
246 | |
247 | ret = ivpu_pll_cmd_send(vdev, min_ratio: hw->pll.min_ratio, max_ratio: hw->pll.max_ratio, |
248 | target_ratio, epp, config, cdyn); |
249 | if (ret) { |
250 | ivpu_err(vdev, "Failed to send PLL workpoint request: %d\n" , ret); |
251 | return ret; |
252 | } |
253 | |
254 | if (enable) { |
255 | ret = ivpu_pll_wait_for_status_ready(vdev); |
256 | if (ret) { |
257 | ivpu_err(vdev, "Timed out waiting for PLL ready status\n" ); |
258 | return ret; |
259 | } |
260 | } |
261 | |
262 | return 0; |
263 | } |
264 | |
265 | static int ivpu_pll_enable(struct ivpu_device *vdev) |
266 | { |
267 | return ivpu_pll_drive(vdev, enable: true); |
268 | } |
269 | |
270 | static int ivpu_pll_disable(struct ivpu_device *vdev) |
271 | { |
272 | return ivpu_pll_drive(vdev, enable: false); |
273 | } |
274 | |
275 | static void ivpu_boot_host_ss_rst_drive(struct ivpu_device *vdev, bool enable) |
276 | { |
277 | u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_RST_EN); |
278 | |
279 | if (enable) { |
280 | val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val); |
281 | val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val); |
282 | val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val); |
283 | } else { |
284 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val); |
285 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val); |
286 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val); |
287 | } |
288 | |
289 | REGV_WR32(VPU_40XX_HOST_SS_CPR_RST_EN, val); |
290 | } |
291 | |
292 | static void ivpu_boot_host_ss_clk_drive(struct ivpu_device *vdev, bool enable) |
293 | { |
294 | u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_CLK_EN); |
295 | |
296 | if (enable) { |
297 | val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val); |
298 | val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val); |
299 | val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val); |
300 | } else { |
301 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val); |
302 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val); |
303 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val); |
304 | } |
305 | |
306 | REGV_WR32(VPU_40XX_HOST_SS_CPR_CLK_EN, val); |
307 | } |
308 | |
309 | static int ivpu_boot_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val) |
310 | { |
311 | u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN); |
312 | |
313 | if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val)) |
314 | return -EIO; |
315 | |
316 | return 0; |
317 | } |
318 | |
319 | static int ivpu_boot_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) |
320 | { |
321 | u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QACCEPTN); |
322 | |
323 | if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val)) |
324 | return -EIO; |
325 | |
326 | return 0; |
327 | } |
328 | |
329 | static int ivpu_boot_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) |
330 | { |
331 | u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QDENY); |
332 | |
333 | if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val)) |
334 | return -EIO; |
335 | |
336 | return 0; |
337 | } |
338 | |
339 | static int ivpu_boot_top_noc_qrenqn_check(struct ivpu_device *vdev, u32 exp_val) |
340 | { |
341 | u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN); |
342 | |
343 | if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) || |
344 | !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val)) |
345 | return -EIO; |
346 | |
347 | return 0; |
348 | } |
349 | |
350 | static int ivpu_boot_top_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) |
351 | { |
352 | u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QACCEPTN); |
353 | |
354 | if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) || |
355 | !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val)) |
356 | return -EIO; |
357 | |
358 | return 0; |
359 | } |
360 | |
361 | static int ivpu_boot_top_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) |
362 | { |
363 | u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QDENY); |
364 | |
365 | if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) || |
366 | !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val)) |
367 | return -EIO; |
368 | |
369 | return 0; |
370 | } |
371 | |
372 | static void ivpu_boot_idle_gen_drive(struct ivpu_device *vdev, bool enable) |
373 | { |
374 | u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_IDLE_GEN); |
375 | |
376 | if (enable) |
377 | val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val); |
378 | else |
379 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val); |
380 | |
381 | REGV_WR32(VPU_40XX_HOST_SS_AON_IDLE_GEN, val); |
382 | } |
383 | |
384 | static int ivpu_boot_host_ss_check(struct ivpu_device *vdev) |
385 | { |
386 | int ret; |
387 | |
388 | ret = ivpu_boot_noc_qreqn_check(vdev, exp_val: 0x0); |
389 | if (ret) { |
390 | ivpu_err(vdev, "Failed qreqn check: %d\n" , ret); |
391 | return ret; |
392 | } |
393 | |
394 | ret = ivpu_boot_noc_qacceptn_check(vdev, exp_val: 0x0); |
395 | if (ret) { |
396 | ivpu_err(vdev, "Failed qacceptn check: %d\n" , ret); |
397 | return ret; |
398 | } |
399 | |
400 | ret = ivpu_boot_noc_qdeny_check(vdev, exp_val: 0x0); |
401 | if (ret) |
402 | ivpu_err(vdev, "Failed qdeny check %d\n" , ret); |
403 | |
404 | return ret; |
405 | } |
406 | |
407 | static int ivpu_boot_host_ss_axi_drive(struct ivpu_device *vdev, bool enable) |
408 | { |
409 | int ret; |
410 | u32 val; |
411 | |
412 | val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN); |
413 | if (enable) |
414 | val = REG_SET_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); |
415 | else |
416 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val); |
417 | REGV_WR32(VPU_40XX_HOST_SS_NOC_QREQN, val); |
418 | |
419 | ret = ivpu_boot_noc_qacceptn_check(vdev, exp_val: enable ? 0x1 : 0x0); |
420 | if (ret) { |
421 | ivpu_err(vdev, "Failed qacceptn check: %d\n" , ret); |
422 | return ret; |
423 | } |
424 | |
425 | ret = ivpu_boot_noc_qdeny_check(vdev, exp_val: 0x0); |
426 | if (ret) { |
427 | ivpu_err(vdev, "Failed qdeny check: %d\n" , ret); |
428 | return ret; |
429 | } |
430 | |
431 | if (enable) { |
432 | REGB_WR32(VPU_40XX_BUTTRESS_PORT_ARBITRATION_WEIGHTS, WEIGHTS_DEFAULT); |
433 | REGB_WR32(VPU_40XX_BUTTRESS_PORT_ARBITRATION_WEIGHTS_ATS, WEIGHTS_ATS_DEFAULT); |
434 | } |
435 | |
436 | return ret; |
437 | } |
438 | |
439 | static int ivpu_boot_host_ss_axi_enable(struct ivpu_device *vdev) |
440 | { |
441 | return ivpu_boot_host_ss_axi_drive(vdev, enable: true); |
442 | } |
443 | |
444 | static int ivpu_boot_host_ss_top_noc_drive(struct ivpu_device *vdev, bool enable) |
445 | { |
446 | int ret; |
447 | u32 val; |
448 | |
449 | val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN); |
450 | if (enable) { |
451 | val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val); |
452 | val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); |
453 | } else { |
454 | val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val); |
455 | val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val); |
456 | } |
457 | REGV_WR32(VPU_40XX_TOP_NOC_QREQN, val); |
458 | |
459 | ret = ivpu_boot_top_noc_qacceptn_check(vdev, exp_val: enable ? 0x1 : 0x0); |
460 | if (ret) { |
461 | ivpu_err(vdev, "Failed qacceptn check: %d\n" , ret); |
462 | return ret; |
463 | } |
464 | |
465 | ret = ivpu_boot_top_noc_qdeny_check(vdev, exp_val: 0x0); |
466 | if (ret) |
467 | ivpu_err(vdev, "Failed qdeny check: %d\n" , ret); |
468 | |
469 | return ret; |
470 | } |
471 | |
472 | static int ivpu_boot_host_ss_top_noc_enable(struct ivpu_device *vdev) |
473 | { |
474 | return ivpu_boot_host_ss_top_noc_drive(vdev, enable: true); |
475 | } |
476 | |
477 | static void ivpu_boot_pwr_island_trickle_drive(struct ivpu_device *vdev, bool enable) |
478 | { |
479 | u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0); |
480 | |
481 | if (enable) |
482 | val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val); |
483 | else |
484 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val); |
485 | |
486 | REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val); |
487 | |
488 | if (enable) |
489 | ndelay(500); |
490 | } |
491 | |
492 | static void ivpu_boot_pwr_island_drive(struct ivpu_device *vdev, bool enable) |
493 | { |
494 | u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0); |
495 | |
496 | if (enable) |
497 | val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val); |
498 | else |
499 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val); |
500 | |
501 | REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val); |
502 | |
503 | if (!enable) |
504 | ndelay(500); |
505 | } |
506 | |
507 | static int ivpu_boot_wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val) |
508 | { |
509 | if (ivpu_is_fpga(vdev)) |
510 | return 0; |
511 | |
512 | return REGV_POLL_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0, CSS_CPU, |
513 | exp_val, PWR_ISLAND_STATUS_TIMEOUT_US); |
514 | } |
515 | |
516 | static void ivpu_boot_pwr_island_isolation_drive(struct ivpu_device *vdev, bool enable) |
517 | { |
518 | u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0); |
519 | |
520 | if (enable) |
521 | val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val); |
522 | else |
523 | val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val); |
524 | |
525 | REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, val); |
526 | } |
527 | |
528 | static void ivpu_boot_no_snoop_enable(struct ivpu_device *vdev) |
529 | { |
530 | u32 val = REGV_RD32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES); |
531 | |
532 | val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, SNOOP_OVERRIDE_EN, val); |
533 | val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AW_SNOOP_OVERRIDE, val); |
534 | val = REG_CLR_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val); |
535 | |
536 | REGV_WR32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, val); |
537 | } |
538 | |
539 | static void ivpu_boot_tbu_mmu_enable(struct ivpu_device *vdev) |
540 | { |
541 | u32 val = REGV_RD32(VPU_40XX_HOST_IF_TBU_MMUSSIDV); |
542 | |
543 | val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val); |
544 | val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val); |
545 | val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_AWMMUSSIDV, val); |
546 | val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_ARMMUSSIDV, val); |
547 | val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val); |
548 | val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val); |
549 | |
550 | REGV_WR32(VPU_40XX_HOST_IF_TBU_MMUSSIDV, val); |
551 | } |
552 | |
553 | static int ivpu_boot_cpu_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val) |
554 | { |
555 | u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN); |
556 | |
557 | if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN, TOP_MMIO, exp_val, val)) |
558 | return -EIO; |
559 | |
560 | return 0; |
561 | } |
562 | |
563 | static int ivpu_boot_cpu_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val) |
564 | { |
565 | u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QDENY); |
566 | |
567 | if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QDENY, TOP_MMIO, exp_val, val)) |
568 | return -EIO; |
569 | |
570 | return 0; |
571 | } |
572 | |
573 | static int ivpu_boot_pwr_domain_enable(struct ivpu_device *vdev) |
574 | { |
575 | int ret; |
576 | |
577 | ret = ivpu_wait_for_clock_own_resource_ack(vdev); |
578 | if (ret) { |
579 | ivpu_err(vdev, "Timed out waiting for clock own resource ACK\n" ); |
580 | return ret; |
581 | } |
582 | |
583 | ivpu_boot_pwr_island_trickle_drive(vdev, enable: true); |
584 | ivpu_boot_pwr_island_drive(vdev, enable: true); |
585 | |
586 | ret = ivpu_boot_wait_for_pwr_island_status(vdev, exp_val: 0x1); |
587 | if (ret) { |
588 | ivpu_err(vdev, "Timed out waiting for power island status\n" ); |
589 | return ret; |
590 | } |
591 | |
592 | ret = ivpu_boot_top_noc_qrenqn_check(vdev, exp_val: 0x0); |
593 | if (ret) { |
594 | ivpu_err(vdev, "Failed qrenqn check %d\n" , ret); |
595 | return ret; |
596 | } |
597 | |
598 | ivpu_boot_host_ss_clk_drive(vdev, enable: true); |
599 | ivpu_boot_host_ss_rst_drive(vdev, enable: true); |
600 | ivpu_boot_pwr_island_isolation_drive(vdev, enable: false); |
601 | |
602 | return ret; |
603 | } |
604 | |
605 | static int ivpu_boot_soc_cpu_drive(struct ivpu_device *vdev, bool enable) |
606 | { |
607 | int ret; |
608 | u32 val; |
609 | |
610 | val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QREQN); |
611 | if (enable) |
612 | val = REG_SET_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val); |
613 | else |
614 | val = REG_CLR_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val); |
615 | REGV_WR32(VPU_40XX_CPU_SS_CPR_NOC_QREQN, val); |
616 | |
617 | ret = ivpu_boot_cpu_noc_qacceptn_check(vdev, exp_val: enable ? 0x1 : 0x0); |
618 | if (ret) { |
619 | ivpu_err(vdev, "Failed qacceptn check: %d\n" , ret); |
620 | return ret; |
621 | } |
622 | |
623 | ret = ivpu_boot_cpu_noc_qdeny_check(vdev, exp_val: 0x0); |
624 | if (ret) |
625 | ivpu_err(vdev, "Failed qdeny check: %d\n" , ret); |
626 | |
627 | return ret; |
628 | } |
629 | |
630 | static int ivpu_boot_soc_cpu_enable(struct ivpu_device *vdev) |
631 | { |
632 | return ivpu_boot_soc_cpu_drive(vdev, enable: true); |
633 | } |
634 | |
635 | static int ivpu_boot_soc_cpu_boot(struct ivpu_device *vdev) |
636 | { |
637 | int ret; |
638 | u32 val; |
639 | u64 val64; |
640 | |
641 | ret = ivpu_boot_soc_cpu_enable(vdev); |
642 | if (ret) { |
643 | ivpu_err(vdev, "Failed to enable SOC CPU: %d\n" , ret); |
644 | return ret; |
645 | } |
646 | |
647 | val64 = vdev->fw->entry_point; |
648 | val64 <<= ffs(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IMAGE_LOCATION_MASK) - 1; |
649 | REGV_WR64(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val64); |
650 | |
651 | val = REGV_RD32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO); |
652 | val = REG_SET_FLD(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, DONE, val); |
653 | REGV_WR32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val); |
654 | |
655 | ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n" , |
656 | ivpu_fw_is_cold_boot(vdev) ? "cold boot" : "resume" ); |
657 | |
658 | return 0; |
659 | } |
660 | |
661 | static int ivpu_boot_d0i3_drive(struct ivpu_device *vdev, bool enable) |
662 | { |
663 | int ret; |
664 | u32 val; |
665 | |
666 | ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); |
667 | if (ret) { |
668 | ivpu_err(vdev, "Failed to sync before D0i3 transition: %d\n" , ret); |
669 | return ret; |
670 | } |
671 | |
672 | val = REGB_RD32(VPU_40XX_BUTTRESS_D0I3_CONTROL); |
673 | if (enable) |
674 | val = REG_SET_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, I3, val); |
675 | else |
676 | val = REG_CLR_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, I3, val); |
677 | REGB_WR32(VPU_40XX_BUTTRESS_D0I3_CONTROL, val); |
678 | |
679 | ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_D0I3_CONTROL, INPROGRESS, 0, TIMEOUT_US); |
680 | if (ret) { |
681 | ivpu_err(vdev, "Failed to sync after D0i3 transition: %d\n" , ret); |
682 | return ret; |
683 | } |
684 | |
685 | return 0; |
686 | } |
687 | |
688 | static bool ivpu_tile_disable_check(u32 config) |
689 | { |
690 | /* Allowed values: 0 or one bit from range 0-5 (6 tiles) */ |
691 | if (config == 0) |
692 | return true; |
693 | |
694 | if (config > BIT(TILE_MAX_NUM - 1)) |
695 | return false; |
696 | |
697 | if ((config & (config - 1)) == 0) |
698 | return true; |
699 | |
700 | return false; |
701 | } |
702 | |
703 | static int ivpu_hw_40xx_info_init(struct ivpu_device *vdev) |
704 | { |
705 | struct ivpu_hw_info *hw = vdev->hw; |
706 | u32 tile_disable; |
707 | u32 fuse; |
708 | |
709 | fuse = REGB_RD32(VPU_40XX_BUTTRESS_TILE_FUSE); |
710 | if (!REG_TEST_FLD(VPU_40XX_BUTTRESS_TILE_FUSE, VALID, fuse)) { |
711 | ivpu_err(vdev, "Fuse: invalid (0x%x)\n" , fuse); |
712 | return -EIO; |
713 | } |
714 | |
715 | tile_disable = REG_GET_FLD(VPU_40XX_BUTTRESS_TILE_FUSE, CONFIG, fuse); |
716 | if (!ivpu_tile_disable_check(config: tile_disable)) { |
717 | ivpu_err(vdev, "Fuse: Invalid tile disable config (0x%x)\n" , tile_disable); |
718 | return -EIO; |
719 | } |
720 | |
721 | if (tile_disable) |
722 | ivpu_dbg(vdev, MISC, "Fuse: %d tiles enabled. Tile number %d disabled\n" , |
723 | TILE_MAX_NUM - 1, ffs(tile_disable) - 1); |
724 | else |
725 | ivpu_dbg(vdev, MISC, "Fuse: All %d tiles enabled\n" , TILE_MAX_NUM); |
726 | |
727 | hw->tile_fuse = tile_disable; |
728 | hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; |
729 | |
730 | ivpu_pll_init_frequency_ratios(vdev); |
731 | |
732 | ivpu_hw_init_range(range: &vdev->hw->ranges.global, start: 0x80000000, SZ_512M); |
733 | ivpu_hw_init_range(range: &vdev->hw->ranges.user, start: 0x80000000, SZ_256M); |
734 | ivpu_hw_init_range(range: &vdev->hw->ranges.shave, start: 0x80000000 + SZ_256M, SZ_2G - SZ_256M); |
735 | ivpu_hw_init_range(range: &vdev->hw->ranges.dma, start: 0x200000000, SZ_8G); |
736 | |
737 | ivpu_hw_read_platform(vdev); |
738 | ivpu_hw_wa_init(vdev); |
739 | ivpu_hw_timeouts_init(vdev); |
740 | |
741 | return 0; |
742 | } |
743 | |
744 | static int ivpu_hw_40xx_ip_reset(struct ivpu_device *vdev) |
745 | { |
746 | int ret; |
747 | u32 val; |
748 | |
749 | ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, 0, TIMEOUT_US); |
750 | if (ret) { |
751 | ivpu_err(vdev, "Wait for *_TRIGGER timed out\n" ); |
752 | return ret; |
753 | } |
754 | |
755 | val = REGB_RD32(VPU_40XX_BUTTRESS_IP_RESET); |
756 | val = REG_SET_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, val); |
757 | REGB_WR32(VPU_40XX_BUTTRESS_IP_RESET, val); |
758 | |
759 | ret = REGB_POLL_FLD(VPU_40XX_BUTTRESS_IP_RESET, TRIGGER, 0, TIMEOUT_US); |
760 | if (ret) |
761 | ivpu_err(vdev, "Timed out waiting for RESET completion\n" ); |
762 | |
763 | return ret; |
764 | } |
765 | |
766 | static int ivpu_hw_40xx_reset(struct ivpu_device *vdev) |
767 | { |
768 | int ret = 0; |
769 | |
770 | if (ivpu_hw_40xx_ip_reset(vdev)) { |
771 | ivpu_err(vdev, "Failed to reset NPU IP\n" ); |
772 | ret = -EIO; |
773 | } |
774 | |
775 | if (ivpu_pll_disable(vdev)) { |
776 | ivpu_err(vdev, "Failed to disable PLL\n" ); |
777 | ret = -EIO; |
778 | } |
779 | |
780 | return ret; |
781 | } |
782 | |
783 | static int ivpu_hw_40xx_d0i3_enable(struct ivpu_device *vdev) |
784 | { |
785 | int ret; |
786 | |
787 | if (IVPU_WA(punit_disabled)) |
788 | return 0; |
789 | |
790 | ret = ivpu_boot_d0i3_drive(vdev, enable: true); |
791 | if (ret) |
792 | ivpu_err(vdev, "Failed to enable D0i3: %d\n" , ret); |
793 | |
794 | udelay(5); /* VPU requires 5 us to complete the transition */ |
795 | |
796 | return ret; |
797 | } |
798 | |
799 | static int ivpu_hw_40xx_d0i3_disable(struct ivpu_device *vdev) |
800 | { |
801 | int ret; |
802 | |
803 | if (IVPU_WA(punit_disabled)) |
804 | return 0; |
805 | |
806 | ret = ivpu_boot_d0i3_drive(vdev, enable: false); |
807 | if (ret) |
808 | ivpu_err(vdev, "Failed to disable D0i3: %d\n" , ret); |
809 | |
810 | return ret; |
811 | } |
812 | |
813 | static void ivpu_hw_40xx_profiling_freq_reg_set(struct ivpu_device *vdev) |
814 | { |
815 | u32 val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS); |
816 | |
817 | if (vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_DEFAULT) |
818 | val = REG_CLR_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, PERF_CLK, val); |
819 | else |
820 | val = REG_SET_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, PERF_CLK, val); |
821 | |
822 | REGB_WR32(VPU_40XX_BUTTRESS_VPU_STATUS, val); |
823 | } |
824 | |
825 | static void ivpu_hw_40xx_ats_print(struct ivpu_device *vdev) |
826 | { |
827 | ivpu_dbg(vdev, MISC, "Buttress ATS: %s\n" , |
828 | REGB_RD32(VPU_40XX_BUTTRESS_HM_ATS) ? "Enable" : "Disable" ); |
829 | } |
830 | |
831 | static void ivpu_hw_40xx_clock_relinquish_disable(struct ivpu_device *vdev) |
832 | { |
833 | u32 val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS); |
834 | |
835 | val = REG_SET_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, DISABLE_CLK_RELINQUISH, val); |
836 | REGB_WR32(VPU_40XX_BUTTRESS_VPU_STATUS, val); |
837 | } |
838 | |
839 | static int ivpu_hw_40xx_power_up(struct ivpu_device *vdev) |
840 | { |
841 | int ret; |
842 | |
843 | ret = ivpu_hw_40xx_d0i3_disable(vdev); |
844 | if (ret) |
845 | ivpu_warn(vdev, "Failed to disable D0I3: %d\n" , ret); |
846 | |
847 | ret = ivpu_pll_enable(vdev); |
848 | if (ret) { |
849 | ivpu_err(vdev, "Failed to enable PLL: %d\n" , ret); |
850 | return ret; |
851 | } |
852 | |
853 | if (IVPU_WA(disable_clock_relinquish)) |
854 | ivpu_hw_40xx_clock_relinquish_disable(vdev); |
855 | ivpu_hw_40xx_profiling_freq_reg_set(vdev); |
856 | ivpu_hw_40xx_ats_print(vdev); |
857 | |
858 | ret = ivpu_boot_host_ss_check(vdev); |
859 | if (ret) { |
860 | ivpu_err(vdev, "Failed to configure host SS: %d\n" , ret); |
861 | return ret; |
862 | } |
863 | |
864 | ivpu_boot_idle_gen_drive(vdev, enable: false); |
865 | |
866 | ret = ivpu_boot_pwr_domain_enable(vdev); |
867 | if (ret) { |
868 | ivpu_err(vdev, "Failed to enable power domain: %d\n" , ret); |
869 | return ret; |
870 | } |
871 | |
872 | ret = ivpu_boot_host_ss_axi_enable(vdev); |
873 | if (ret) { |
874 | ivpu_err(vdev, "Failed to enable AXI: %d\n" , ret); |
875 | return ret; |
876 | } |
877 | |
878 | ret = ivpu_boot_host_ss_top_noc_enable(vdev); |
879 | if (ret) |
880 | ivpu_err(vdev, "Failed to enable TOP NOC: %d\n" , ret); |
881 | |
882 | return ret; |
883 | } |
884 | |
885 | static int ivpu_hw_40xx_boot_fw(struct ivpu_device *vdev) |
886 | { |
887 | int ret; |
888 | |
889 | ivpu_boot_no_snoop_enable(vdev); |
890 | ivpu_boot_tbu_mmu_enable(vdev); |
891 | |
892 | ret = ivpu_boot_soc_cpu_boot(vdev); |
893 | if (ret) |
894 | ivpu_err(vdev, "Failed to boot SOC CPU: %d\n" , ret); |
895 | |
896 | return ret; |
897 | } |
898 | |
899 | static bool ivpu_hw_40xx_is_idle(struct ivpu_device *vdev) |
900 | { |
901 | u32 val; |
902 | |
903 | if (IVPU_WA(punit_disabled)) |
904 | return true; |
905 | |
906 | val = REGB_RD32(VPU_40XX_BUTTRESS_VPU_STATUS); |
907 | return REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, READY, val) && |
908 | REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, IDLE, val); |
909 | } |
910 | |
911 | static int ivpu_hw_40xx_wait_for_idle(struct ivpu_device *vdev) |
912 | { |
913 | return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US); |
914 | } |
915 | |
916 | static void ivpu_hw_40xx_save_d0i3_entry_timestamp(struct ivpu_device *vdev) |
917 | { |
918 | vdev->hw->d0i3_entry_host_ts = ktime_get_boottime(); |
919 | vdev->hw->d0i3_entry_vpu_ts = REGV_RD64(VPU_40XX_CPU_SS_TIM_PERF_EXT_FREE_CNT); |
920 | } |
921 | |
922 | static int ivpu_hw_40xx_power_down(struct ivpu_device *vdev) |
923 | { |
924 | int ret = 0; |
925 | |
926 | ivpu_hw_40xx_save_d0i3_entry_timestamp(vdev); |
927 | |
928 | if (!ivpu_hw_40xx_is_idle(vdev) && ivpu_hw_40xx_ip_reset(vdev)) |
929 | ivpu_warn(vdev, "Failed to reset the NPU\n" ); |
930 | |
931 | if (ivpu_pll_disable(vdev)) { |
932 | ivpu_err(vdev, "Failed to disable PLL\n" ); |
933 | ret = -EIO; |
934 | } |
935 | |
936 | if (ivpu_hw_40xx_d0i3_enable(vdev)) { |
937 | ivpu_err(vdev, "Failed to enter D0I3\n" ); |
938 | ret = -EIO; |
939 | } |
940 | |
941 | return ret; |
942 | } |
943 | |
944 | static void ivpu_hw_40xx_wdt_disable(struct ivpu_device *vdev) |
945 | { |
946 | u32 val; |
947 | |
948 | REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); |
949 | REGV_WR32(VPU_40XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE); |
950 | |
951 | REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE); |
952 | REGV_WR32(VPU_40XX_CPU_SS_TIM_WDOG_EN, 0); |
953 | |
954 | val = REGV_RD32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG); |
955 | val = REG_CLR_FLD(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val); |
956 | REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val); |
957 | } |
958 | |
959 | static u32 ivpu_hw_40xx_profiling_freq_get(struct ivpu_device *vdev) |
960 | { |
961 | return vdev->hw->pll.profiling_freq; |
962 | } |
963 | |
964 | static void ivpu_hw_40xx_profiling_freq_drive(struct ivpu_device *vdev, bool enable) |
965 | { |
966 | if (enable) |
967 | vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH; |
968 | else |
969 | vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT; |
970 | } |
971 | |
972 | /* Register indirect accesses */ |
973 | static u32 ivpu_hw_40xx_reg_pll_freq_get(struct ivpu_device *vdev) |
974 | { |
975 | u32 pll_curr_ratio; |
976 | |
977 | pll_curr_ratio = REGB_RD32(VPU_40XX_BUTTRESS_PLL_FREQ); |
978 | pll_curr_ratio &= VPU_40XX_BUTTRESS_PLL_FREQ_RATIO_MASK; |
979 | |
980 | return PLL_RATIO_TO_FREQ(pll_curr_ratio); |
981 | } |
982 | |
983 | static u32 ivpu_hw_40xx_ratio_to_freq(struct ivpu_device *vdev, u32 ratio) |
984 | { |
985 | return PLL_RATIO_TO_FREQ(ratio); |
986 | } |
987 | |
988 | static u32 ivpu_hw_40xx_reg_telemetry_offset_get(struct ivpu_device *vdev) |
989 | { |
990 | return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_OFFSET); |
991 | } |
992 | |
993 | static u32 ivpu_hw_40xx_reg_telemetry_size_get(struct ivpu_device *vdev) |
994 | { |
995 | return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_SIZE); |
996 | } |
997 | |
998 | static u32 ivpu_hw_40xx_reg_telemetry_enable_get(struct ivpu_device *vdev) |
999 | { |
1000 | return REGB_RD32(VPU_40XX_BUTTRESS_VPU_TELEMETRY_ENABLE); |
1001 | } |
1002 | |
1003 | static void ivpu_hw_40xx_reg_db_set(struct ivpu_device *vdev, u32 db_id) |
1004 | { |
1005 | u32 reg_stride = VPU_40XX_CPU_SS_DOORBELL_1 - VPU_40XX_CPU_SS_DOORBELL_0; |
1006 | u32 val = REG_FLD(VPU_40XX_CPU_SS_DOORBELL_0, SET); |
1007 | |
1008 | REGV_WR32I(VPU_40XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val); |
1009 | } |
1010 | |
1011 | static u32 ivpu_hw_40xx_reg_ipc_rx_addr_get(struct ivpu_device *vdev) |
1012 | { |
1013 | return REGV_RD32(VPU_40XX_HOST_SS_TIM_IPC_FIFO_ATM); |
1014 | } |
1015 | |
1016 | static u32 ivpu_hw_40xx_reg_ipc_rx_count_get(struct ivpu_device *vdev) |
1017 | { |
1018 | u32 count = REGV_RD32_SILENT(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT); |
1019 | |
1020 | return REG_GET_FLD(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count); |
1021 | } |
1022 | |
1023 | static void ivpu_hw_40xx_reg_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr) |
1024 | { |
1025 | REGV_WR32(VPU_40XX_CPU_SS_TIM_IPC_FIFO, vpu_addr); |
1026 | } |
1027 | |
1028 | static void ivpu_hw_40xx_irq_clear(struct ivpu_device *vdev) |
1029 | { |
1030 | REGV_WR64(VPU_40XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK); |
1031 | } |
1032 | |
1033 | static void ivpu_hw_40xx_irq_enable(struct ivpu_device *vdev) |
1034 | { |
1035 | REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK); |
1036 | REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK); |
1037 | REGB_WR32(VPU_40XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_ENABLE_MASK); |
1038 | REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0); |
1039 | } |
1040 | |
1041 | static void ivpu_hw_40xx_irq_disable(struct ivpu_device *vdev) |
1042 | { |
1043 | REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1); |
1044 | REGB_WR32(VPU_40XX_BUTTRESS_LOCAL_INT_MASK, BUTTRESS_IRQ_DISABLE_MASK); |
1045 | REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, 0x0ull); |
1046 | REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, 0x0ul); |
1047 | } |
1048 | |
1049 | static void ivpu_hw_40xx_irq_wdt_nce_handler(struct ivpu_device *vdev) |
1050 | { |
1051 | /* TODO: For LNN hang consider engine reset instead of full recovery */ |
1052 | ivpu_pm_trigger_recovery(vdev, reason: "WDT NCE IRQ" ); |
1053 | } |
1054 | |
1055 | static void ivpu_hw_40xx_irq_wdt_mss_handler(struct ivpu_device *vdev) |
1056 | { |
1057 | ivpu_hw_wdt_disable(vdev); |
1058 | ivpu_pm_trigger_recovery(vdev, reason: "WDT MSS IRQ" ); |
1059 | } |
1060 | |
1061 | static void ivpu_hw_40xx_irq_noc_firewall_handler(struct ivpu_device *vdev) |
1062 | { |
1063 | ivpu_pm_trigger_recovery(vdev, reason: "NOC Firewall IRQ" ); |
1064 | } |
1065 | |
1066 | /* Handler for IRQs from VPU core (irqV) */ |
1067 | static bool ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq, bool *wake_thread) |
1068 | { |
1069 | u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK; |
1070 | |
1071 | if (!status) |
1072 | return false; |
1073 | |
1074 | REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status); |
1075 | |
1076 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status)) |
1077 | ivpu_mmu_irq_evtq_handler(vdev); |
1078 | |
1079 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status)) |
1080 | ivpu_ipc_irq_handler(vdev, wake_thread); |
1081 | |
1082 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status)) |
1083 | ivpu_dbg(vdev, IRQ, "MMU sync complete\n" ); |
1084 | |
1085 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status)) |
1086 | ivpu_mmu_irq_gerr_handler(vdev); |
1087 | |
1088 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status)) |
1089 | ivpu_hw_40xx_irq_wdt_mss_handler(vdev); |
1090 | |
1091 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status)) |
1092 | ivpu_hw_40xx_irq_wdt_nce_handler(vdev); |
1093 | |
1094 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status)) |
1095 | ivpu_hw_40xx_irq_noc_firewall_handler(vdev); |
1096 | |
1097 | return true; |
1098 | } |
1099 | |
1100 | /* Handler for IRQs from Buttress core (irqB) */ |
1101 | static bool ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq) |
1102 | { |
1103 | bool schedule_recovery = false; |
1104 | u32 status = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK; |
1105 | |
1106 | if (!status) |
1107 | return false; |
1108 | |
1109 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE, status)) |
1110 | ivpu_dbg(vdev, IRQ, "FREQ_CHANGE" ); |
1111 | |
1112 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, status)) { |
1113 | ivpu_err(vdev, "ATS_ERR LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n" , |
1114 | REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG1), |
1115 | REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG2)); |
1116 | REGB_WR32(VPU_40XX_BUTTRESS_ATS_ERR_CLEAR, 0x1); |
1117 | schedule_recovery = true; |
1118 | } |
1119 | |
1120 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR, status)) { |
1121 | ivpu_err(vdev, "CFI0_ERR 0x%08x" , REGB_RD32(VPU_40XX_BUTTRESS_CFI0_ERR_LOG)); |
1122 | REGB_WR32(VPU_40XX_BUTTRESS_CFI0_ERR_CLEAR, 0x1); |
1123 | schedule_recovery = true; |
1124 | } |
1125 | |
1126 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR, status)) { |
1127 | ivpu_err(vdev, "CFI1_ERR 0x%08x" , REGB_RD32(VPU_40XX_BUTTRESS_CFI1_ERR_LOG)); |
1128 | REGB_WR32(VPU_40XX_BUTTRESS_CFI1_ERR_CLEAR, 0x1); |
1129 | schedule_recovery = true; |
1130 | } |
1131 | |
1132 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR, status)) { |
1133 | ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x" , |
1134 | REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_LOW), |
1135 | REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_HIGH)); |
1136 | REGB_WR32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_CLEAR, 0x1); |
1137 | schedule_recovery = true; |
1138 | } |
1139 | |
1140 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR, status)) { |
1141 | ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x" , |
1142 | REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_LOW), |
1143 | REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_HIGH)); |
1144 | REGB_WR32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_CLEAR, 0x1); |
1145 | schedule_recovery = true; |
1146 | } |
1147 | |
1148 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR, status)) { |
1149 | ivpu_err(vdev, "Survivability error detected\n" ); |
1150 | schedule_recovery = true; |
1151 | } |
1152 | |
1153 | /* This must be done after interrupts are cleared at the source. */ |
1154 | REGB_WR32(VPU_40XX_BUTTRESS_INTERRUPT_STAT, status); |
1155 | |
1156 | if (schedule_recovery) |
1157 | ivpu_pm_trigger_recovery(vdev, reason: "Buttress IRQ" ); |
1158 | |
1159 | return true; |
1160 | } |
1161 | |
1162 | static irqreturn_t ivpu_hw_40xx_irq_handler(int irq, void *ptr) |
1163 | { |
1164 | bool irqv_handled, irqb_handled, wake_thread = false; |
1165 | struct ivpu_device *vdev = ptr; |
1166 | |
1167 | REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1); |
1168 | |
1169 | irqv_handled = ivpu_hw_40xx_irqv_handler(vdev, irq, wake_thread: &wake_thread); |
1170 | irqb_handled = ivpu_hw_40xx_irqb_handler(vdev, irq); |
1171 | |
1172 | /* Re-enable global interrupts to re-trigger MSI for pending interrupts */ |
1173 | REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0); |
1174 | |
1175 | if (wake_thread) |
1176 | return IRQ_WAKE_THREAD; |
1177 | if (irqv_handled || irqb_handled) |
1178 | return IRQ_HANDLED; |
1179 | return IRQ_NONE; |
1180 | } |
1181 | |
1182 | static void ivpu_hw_40xx_diagnose_failure(struct ivpu_device *vdev) |
1183 | { |
1184 | u32 irqv = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK; |
1185 | u32 irqb = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK; |
1186 | |
1187 | if (ivpu_hw_40xx_reg_ipc_rx_count_get(vdev)) |
1188 | ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ" ); |
1189 | |
1190 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, irqv)) |
1191 | ivpu_err(vdev, "WDT MSS timeout detected\n" ); |
1192 | |
1193 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, irqv)) |
1194 | ivpu_err(vdev, "WDT NCE timeout detected\n" ); |
1195 | |
1196 | if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, irqv)) |
1197 | ivpu_err(vdev, "NOC Firewall irq detected\n" ); |
1198 | |
1199 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, ATS_ERR, irqb)) { |
1200 | ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n" , |
1201 | REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG1), |
1202 | REGB_RD32(VPU_40XX_BUTTRESS_ATS_ERR_LOG2)); |
1203 | } |
1204 | |
1205 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI0_ERR, irqb)) |
1206 | ivpu_err(vdev, "CFI0_ERR_LOG 0x%08x\n" , REGB_RD32(VPU_40XX_BUTTRESS_CFI0_ERR_LOG)); |
1207 | |
1208 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, CFI1_ERR, irqb)) |
1209 | ivpu_err(vdev, "CFI1_ERR_LOG 0x%08x\n" , REGB_RD32(VPU_40XX_BUTTRESS_CFI1_ERR_LOG)); |
1210 | |
1211 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR0_ERR, irqb)) |
1212 | ivpu_err(vdev, "IMR_ERR_CFI0 LOW: 0x%08x HIGH: 0x%08x\n" , |
1213 | REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_LOW), |
1214 | REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI0_HIGH)); |
1215 | |
1216 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, IMR1_ERR, irqb)) |
1217 | ivpu_err(vdev, "IMR_ERR_CFI1 LOW: 0x%08x HIGH: 0x%08x\n" , |
1218 | REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_LOW), |
1219 | REGB_RD32(VPU_40XX_BUTTRESS_IMR_ERR_CFI1_HIGH)); |
1220 | |
1221 | if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, SURV_ERR, irqb)) |
1222 | ivpu_err(vdev, "Survivability error detected\n" ); |
1223 | } |
1224 | |
1225 | const struct ivpu_hw_ops ivpu_hw_40xx_ops = { |
1226 | .info_init = ivpu_hw_40xx_info_init, |
1227 | .power_up = ivpu_hw_40xx_power_up, |
1228 | .is_idle = ivpu_hw_40xx_is_idle, |
1229 | .wait_for_idle = ivpu_hw_40xx_wait_for_idle, |
1230 | .power_down = ivpu_hw_40xx_power_down, |
1231 | .reset = ivpu_hw_40xx_reset, |
1232 | .boot_fw = ivpu_hw_40xx_boot_fw, |
1233 | .wdt_disable = ivpu_hw_40xx_wdt_disable, |
1234 | .diagnose_failure = ivpu_hw_40xx_diagnose_failure, |
1235 | .profiling_freq_get = ivpu_hw_40xx_profiling_freq_get, |
1236 | .profiling_freq_drive = ivpu_hw_40xx_profiling_freq_drive, |
1237 | .reg_pll_freq_get = ivpu_hw_40xx_reg_pll_freq_get, |
1238 | .ratio_to_freq = ivpu_hw_40xx_ratio_to_freq, |
1239 | .reg_telemetry_offset_get = ivpu_hw_40xx_reg_telemetry_offset_get, |
1240 | .reg_telemetry_size_get = ivpu_hw_40xx_reg_telemetry_size_get, |
1241 | .reg_telemetry_enable_get = ivpu_hw_40xx_reg_telemetry_enable_get, |
1242 | .reg_db_set = ivpu_hw_40xx_reg_db_set, |
1243 | .reg_ipc_rx_addr_get = ivpu_hw_40xx_reg_ipc_rx_addr_get, |
1244 | .reg_ipc_rx_count_get = ivpu_hw_40xx_reg_ipc_rx_count_get, |
1245 | .reg_ipc_tx_set = ivpu_hw_40xx_reg_ipc_tx_set, |
1246 | .irq_clear = ivpu_hw_40xx_irq_clear, |
1247 | .irq_enable = ivpu_hw_40xx_irq_enable, |
1248 | .irq_disable = ivpu_hw_40xx_irq_disable, |
1249 | .irq_handler = ivpu_hw_40xx_irq_handler, |
1250 | }; |
1251 | |