1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020 - 2024 Intel Corporation
4 */
5
6#include "ivpu_drv.h"
7#include "ivpu_hw.h"
8#include "ivpu_hw_btrs.h"
9#include "ivpu_hw_ip.h"
10
11#include <asm/msr-index.h>
12#include <asm/msr.h>
13#include <linux/dmi.h>
14#include <linux/fault-inject.h>
15#include <linux/pm_runtime.h>
16
17#ifdef CONFIG_FAULT_INJECTION
18DECLARE_FAULT_ATTR(ivpu_hw_failure);
19
20static char *ivpu_fail_hw;
21module_param_named_unsafe(fail_hw, ivpu_fail_hw, charp, 0444);
22MODULE_PARM_DESC(fail_hw, "<interval>,<probability>,<space>,<times>");
23#endif
24
25#define FW_SHARED_MEM_ALIGNMENT SZ_512K /* VPU MTRR limitation */
26
27#define ECC_MCA_SIGNAL_ENABLE_MASK 0xff
28
29static char *platform_to_str(u32 platform)
30{
31 switch (platform) {
32 case IVPU_PLATFORM_SILICON:
33 return "SILICON";
34 case IVPU_PLATFORM_SIMICS:
35 return "SIMICS";
36 case IVPU_PLATFORM_FPGA:
37 return "FPGA";
38 case IVPU_PLATFORM_HSLE:
39 return "HSLE";
40 default:
41 return "Invalid platform";
42 }
43}
44
45static void platform_init(struct ivpu_device *vdev)
46{
47 int platform = ivpu_hw_btrs_platform_read(vdev);
48
49 ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n", platform_to_str(platform), platform);
50
51 switch (platform) {
52 case IVPU_PLATFORM_SILICON:
53 case IVPU_PLATFORM_SIMICS:
54 case IVPU_PLATFORM_FPGA:
55 case IVPU_PLATFORM_HSLE:
56 vdev->platform = platform;
57 break;
58
59 default:
60 ivpu_err(vdev, "Invalid platform type: %d\n", platform);
61 break;
62 }
63}
64
65static void wa_init(struct ivpu_device *vdev)
66{
67 vdev->wa.punit_disabled = false;
68 vdev->wa.clear_runtime_mem = false;
69
70 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
71 vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev);
72
73 if (ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL &&
74 ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0)
75 vdev->wa.disable_clock_relinquish = true;
76
77 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_ENABLE)
78 vdev->wa.disable_clock_relinquish = false;
79
80 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_DISABLE)
81 vdev->wa.disable_clock_relinquish = true;
82
83 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
84 vdev->wa.wp0_during_power_up = true;
85
86 if (ivpu_test_mode & IVPU_TEST_MODE_D0I2_DISABLE)
87 vdev->wa.disable_d0i2 = true;
88
89 IVPU_PRINT_WA(punit_disabled);
90 IVPU_PRINT_WA(clear_runtime_mem);
91 IVPU_PRINT_WA(interrupt_clear_with_0);
92 IVPU_PRINT_WA(disable_clock_relinquish);
93 IVPU_PRINT_WA(wp0_during_power_up);
94 IVPU_PRINT_WA(disable_d0i2);
95}
96
97static void timeouts_init(struct ivpu_device *vdev)
98{
99 if (ivpu_test_mode & IVPU_TEST_MODE_DISABLE_TIMEOUTS) {
100 vdev->timeout.boot = -1;
101 vdev->timeout.jsm = -1;
102 vdev->timeout.tdr = -1;
103 vdev->timeout.inference = -1;
104 vdev->timeout.autosuspend = -1;
105 vdev->timeout.d0i3_entry_msg = -1;
106 } else if (ivpu_is_fpga(vdev)) {
107 vdev->timeout.boot = 50;
108 vdev->timeout.jsm = 15000;
109 vdev->timeout.tdr = 30000;
110 vdev->timeout.inference = 900000;
111 vdev->timeout.autosuspend = -1;
112 vdev->timeout.d0i3_entry_msg = 500;
113 vdev->timeout.state_dump_msg = 10000;
114 } else if (ivpu_is_simics(vdev)) {
115 vdev->timeout.boot = 50;
116 vdev->timeout.jsm = 500;
117 vdev->timeout.tdr = 10000;
118 vdev->timeout.inference = 300000;
119 vdev->timeout.autosuspend = 100;
120 vdev->timeout.d0i3_entry_msg = 100;
121 vdev->timeout.state_dump_msg = 10;
122 } else {
123 vdev->timeout.boot = 1000;
124 vdev->timeout.jsm = 500;
125 vdev->timeout.tdr = 2000;
126 vdev->timeout.inference = 60000;
127 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
128 vdev->timeout.autosuspend = 10;
129 else
130 vdev->timeout.autosuspend = 100;
131 vdev->timeout.d0i3_entry_msg = 5;
132 vdev->timeout.state_dump_msg = 100;
133 }
134}
135
136static void priority_bands_init(struct ivpu_device *vdev)
137{
138 /* Idle */
139 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 0;
140 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 50000;
141 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 160000;
142 /* Normal */
143 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000;
144 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000;
145 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 300000;
146 /* Focus */
147 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000;
148 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000;
149 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 200000;
150 /* Realtime */
151 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 0;
152 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 50000;
153 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 200000;
154}
155
156int ivpu_hw_range_init(struct ivpu_device *vdev, struct ivpu_addr_range *range, u64 start, u64 size)
157{
158 u64 end;
159
160 if (!range || check_add_overflow(start, size, &end)) {
161 ivpu_err(vdev, "Invalid range: start 0x%llx size %llu\n", start, size);
162 return -EINVAL;
163 }
164
165 range->start = start;
166 range->end = end;
167
168 return 0;
169}
170
171static void memory_ranges_init(struct ivpu_device *vdev)
172{
173 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
174 ivpu_hw_range_init(vdev, range: &vdev->hw->ranges.runtime, start: 0x84800000, SZ_64M);
175 ivpu_hw_range_init(vdev, range: &vdev->hw->ranges.global, start: 0x90000000, SZ_256M);
176 ivpu_hw_range_init(vdev, range: &vdev->hw->ranges.user, start: 0xa0000000, size: 511 * SZ_1M);
177 ivpu_hw_range_init(vdev, range: &vdev->hw->ranges.shave, start: 0x180000000, SZ_2G);
178 ivpu_hw_range_init(vdev, range: &vdev->hw->ranges.dma, start: 0x200000000, SZ_128G);
179 } else {
180 ivpu_hw_range_init(vdev, range: &vdev->hw->ranges.runtime, start: 0x80000000, SZ_64M);
181 ivpu_hw_range_init(vdev, range: &vdev->hw->ranges.global, start: 0x90000000, SZ_256M);
182 ivpu_hw_range_init(vdev, range: &vdev->hw->ranges.shave, start: 0x80000000, SZ_2G);
183 ivpu_hw_range_init(vdev, range: &vdev->hw->ranges.user, start: 0x100000000, SZ_256G);
184 vdev->hw->ranges.dma = vdev->hw->ranges.user;
185 }
186
187 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vdev->hw->ranges.global.start,
188 FW_SHARED_MEM_ALIGNMENT));
189}
190
191static int wp_enable(struct ivpu_device *vdev)
192{
193 return ivpu_hw_btrs_wp_drive(vdev, enable: true);
194}
195
196static int wp_disable(struct ivpu_device *vdev)
197{
198 return ivpu_hw_btrs_wp_drive(vdev, enable: false);
199}
200
201int ivpu_hw_power_up(struct ivpu_device *vdev)
202{
203 int ret;
204
205 if (IVPU_WA(wp0_during_power_up)) {
206 /* WP requests may fail when powering down, so issue WP 0 here */
207 ret = wp_disable(vdev);
208 if (ret)
209 ivpu_warn(vdev, "Failed to disable workpoint: %d\n", ret);
210 }
211
212 ret = ivpu_hw_btrs_d0i3_disable(vdev);
213 if (ret)
214 ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
215
216 ret = wp_enable(vdev);
217 if (ret) {
218 ivpu_err(vdev, "Failed to enable workpoint: %d\n", ret);
219 return ret;
220 }
221
222 if (ivpu_hw_btrs_gen(vdev) >= IVPU_HW_BTRS_LNL) {
223 if (IVPU_WA(disable_clock_relinquish))
224 ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev);
225 ivpu_hw_btrs_profiling_freq_reg_set_lnl(vdev);
226 ivpu_hw_btrs_ats_print_lnl(vdev);
227 }
228
229 ret = ivpu_hw_ip_host_ss_configure(vdev);
230 if (ret) {
231 ivpu_err(vdev, "Failed to configure host SS: %d\n", ret);
232 return ret;
233 }
234
235 ivpu_hw_ip_idle_gen_disable(vdev);
236
237 ret = ivpu_hw_btrs_wait_for_clock_res_own_ack(vdev);
238 if (ret) {
239 ivpu_err(vdev, "Timed out waiting for clock resource own ACK\n");
240 return ret;
241 }
242
243 ret = ivpu_hw_ip_pwr_domain_enable(vdev);
244 if (ret) {
245 ivpu_err(vdev, "Failed to enable power domain: %d\n", ret);
246 return ret;
247 }
248
249 ret = ivpu_hw_ip_host_ss_axi_enable(vdev);
250 if (ret) {
251 ivpu_err(vdev, "Failed to enable AXI: %d\n", ret);
252 return ret;
253 }
254
255 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_LNL)
256 ivpu_hw_btrs_set_port_arbitration_weights_lnl(vdev);
257
258 ret = ivpu_hw_ip_top_noc_enable(vdev);
259 if (ret)
260 ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret);
261
262 return ret;
263}
264
265static void save_d0i3_entry_timestamp(struct ivpu_device *vdev)
266{
267 vdev->hw->d0i3_entry_host_ts = ktime_get_boottime();
268 vdev->hw->d0i3_entry_vpu_ts = ivpu_hw_ip_read_perf_timer_counter(vdev);
269}
270
271int ivpu_hw_reset(struct ivpu_device *vdev)
272{
273 int ret = 0;
274
275 if (ivpu_hw_btrs_ip_reset(vdev)) {
276 ivpu_err(vdev, "Failed to reset NPU IP\n");
277 ret = -EIO;
278 }
279
280 if (wp_disable(vdev)) {
281 ivpu_err(vdev, "Failed to disable workpoint\n");
282 ret = -EIO;
283 }
284
285 return ret;
286}
287
288int ivpu_hw_power_down(struct ivpu_device *vdev)
289{
290 int ret = 0;
291
292 save_d0i3_entry_timestamp(vdev);
293
294 if (!ivpu_hw_is_idle(vdev))
295 ivpu_warn(vdev, "NPU not idle during power down\n");
296
297 if (ivpu_hw_reset(vdev)) {
298 ivpu_err(vdev, "Failed to reset NPU\n");
299 ret = -EIO;
300 }
301
302 if (ivpu_hw_btrs_d0i3_enable(vdev)) {
303 ivpu_err(vdev, "Failed to enter D0I3\n");
304 ret = -EIO;
305 }
306
307 return ret;
308}
309
310int ivpu_hw_init(struct ivpu_device *vdev)
311{
312 ivpu_hw_btrs_info_init(vdev);
313 ivpu_hw_btrs_freq_ratios_init(vdev);
314 priority_bands_init(vdev);
315 memory_ranges_init(vdev);
316 platform_init(vdev);
317 wa_init(vdev);
318 timeouts_init(vdev);
319 atomic_set(v: &vdev->hw->firewall_irq_counter, i: 0);
320
321#ifdef CONFIG_FAULT_INJECTION
322 if (ivpu_fail_hw)
323 setup_fault_attr(attr: &ivpu_hw_failure, str: ivpu_fail_hw);
324#endif
325
326 return 0;
327}
328
329int ivpu_hw_boot_fw(struct ivpu_device *vdev)
330{
331 int ret;
332
333 ivpu_hw_ip_snoop_disable(vdev);
334 ivpu_hw_ip_tbu_mmu_enable(vdev);
335 ret = ivpu_hw_ip_soc_cpu_boot(vdev);
336 if (ret)
337 ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret);
338
339 return ret;
340}
341
342void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
343{
344 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
345 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
346 return;
347 }
348
349 if (enable)
350 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH;
351 else
352 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
353}
354
355void ivpu_irq_handlers_init(struct ivpu_device *vdev)
356{
357 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
358 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_37xx;
359 else
360 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_40xx;
361
362 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
363 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_mtl;
364 else
365 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_lnl;
366}
367
368void ivpu_hw_irq_enable(struct ivpu_device *vdev)
369{
370 ivpu_hw_ip_irq_enable(vdev);
371 ivpu_hw_btrs_irq_enable(vdev);
372}
373
374void ivpu_hw_irq_disable(struct ivpu_device *vdev)
375{
376 ivpu_hw_btrs_irq_disable(vdev);
377 ivpu_hw_ip_irq_disable(vdev);
378}
379
380irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr)
381{
382 struct ivpu_device *vdev = ptr;
383 bool ip_handled, btrs_handled;
384
385 ivpu_hw_btrs_global_int_disable(vdev);
386
387 btrs_handled = ivpu_hw_btrs_irq_handler(vdev, irq);
388 if (!ivpu_hw_is_idle((vdev)) || !btrs_handled)
389 ip_handled = ivpu_hw_ip_irq_handler(vdev, irq);
390 else
391 ip_handled = false;
392
393 /* Re-enable global interrupts to re-trigger MSI for pending interrupts */
394 ivpu_hw_btrs_global_int_enable(vdev);
395
396 if (!ip_handled && !btrs_handled)
397 return IRQ_NONE;
398
399 pm_runtime_mark_last_busy(dev: vdev->drm.dev);
400 return IRQ_HANDLED;
401}
402
403bool ivpu_hw_uses_ecc_mca_signal(struct ivpu_device *vdev)
404{
405 unsigned long long msr_integrity_caps;
406 int ret;
407
408 if (ivpu_hw_ip_gen(vdev) < IVPU_HW_IP_50XX)
409 return false;
410
411 ret = rdmsrq_safe(MSR_INTEGRITY_CAPS, p: &msr_integrity_caps);
412 if (ret) {
413 ivpu_warn(vdev, "Error reading MSR_INTEGRITY_CAPS: %d", ret);
414 return false;
415 }
416
417 ivpu_dbg(vdev, MISC, "MSR_INTEGRITY_CAPS: 0x%llx\n", msr_integrity_caps);
418
419 return msr_integrity_caps & ECC_MCA_SIGNAL_ENABLE_MASK;
420}
421

source code of linux/drivers/accel/ivpu/ivpu_hw.c