1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright 2017, Nicholas Piggin, IBM Corporation |
4 | */ |
5 | |
6 | #define pr_fmt(fmt) "dt-cpu-ftrs: " fmt |
7 | |
8 | #include <linux/export.h> |
9 | #include <linux/init.h> |
10 | #include <linux/jump_label.h> |
11 | #include <linux/libfdt.h> |
12 | #include <linux/memblock.h> |
13 | #include <linux/of_fdt.h> |
14 | #include <linux/printk.h> |
15 | #include <linux/sched.h> |
16 | #include <linux/string.h> |
17 | #include <linux/threads.h> |
18 | |
19 | #include <asm/cputable.h> |
20 | #include <asm/dt_cpu_ftrs.h> |
21 | #include <asm/mce.h> |
22 | #include <asm/mmu.h> |
23 | #include <asm/setup.h> |
24 | |
25 | |
26 | /* Device-tree visible constants follow */ |
27 | #define ISA_V3_0B 3000 |
28 | #define ISA_V3_1 3100 |
29 | |
30 | #define USABLE_PR (1U << 0) |
31 | #define USABLE_OS (1U << 1) |
32 | #define USABLE_HV (1U << 2) |
33 | |
34 | #define HV_SUPPORT_HFSCR (1U << 0) |
35 | #define OS_SUPPORT_FSCR (1U << 0) |
36 | |
37 | /* For parsing, we define all bits set as "NONE" case */ |
38 | #define HV_SUPPORT_NONE 0xffffffffU |
39 | #define OS_SUPPORT_NONE 0xffffffffU |
40 | |
41 | struct dt_cpu_feature { |
42 | const char *name; |
43 | uint32_t isa; |
44 | uint32_t usable_privilege; |
45 | uint32_t hv_support; |
46 | uint32_t os_support; |
47 | uint32_t hfscr_bit_nr; |
48 | uint32_t fscr_bit_nr; |
49 | uint32_t hwcap_bit_nr; |
50 | /* fdt parsing */ |
51 | unsigned long node; |
52 | int enabled; |
53 | int disabled; |
54 | }; |
55 | |
56 | #define MMU_FTRS_HASH_BASE (MMU_FTRS_POWER8) |
57 | |
58 | #define COMMON_USER_BASE (PPC_FEATURE_32 | PPC_FEATURE_64 | \ |
59 | PPC_FEATURE_ARCH_2_06 |\ |
60 | PPC_FEATURE_ICACHE_SNOOP) |
61 | #define COMMON_USER2_BASE (PPC_FEATURE2_ARCH_2_07 | \ |
62 | PPC_FEATURE2_ISEL) |
63 | /* |
64 | * Set up the base CPU |
65 | */ |
66 | |
67 | static int hv_mode; |
68 | |
69 | static struct { |
70 | u64 lpcr; |
71 | u64 hfscr; |
72 | u64 fscr; |
73 | u64 pcr; |
74 | } system_registers; |
75 | |
76 | static void (*init_pmu_registers)(void); |
77 | |
78 | static void __restore_cpu_cpufeatures(void) |
79 | { |
80 | mtspr(SPRN_LPCR, system_registers.lpcr); |
81 | if (hv_mode) { |
82 | mtspr(SPRN_LPID, 0); |
83 | mtspr(SPRN_AMOR, ~0); |
84 | mtspr(SPRN_HFSCR, system_registers.hfscr); |
85 | mtspr(SPRN_PCR, system_registers.pcr); |
86 | } |
87 | mtspr(SPRN_FSCR, system_registers.fscr); |
88 | |
89 | if (init_pmu_registers) |
90 | init_pmu_registers(); |
91 | } |
92 | |
93 | static char dt_cpu_name[64]; |
94 | |
95 | static struct cpu_spec __initdata base_cpu_spec = { |
96 | .cpu_name = NULL, |
97 | .cpu_features = CPU_FTRS_DT_CPU_BASE, |
98 | .cpu_user_features = COMMON_USER_BASE, |
99 | .cpu_user_features2 = COMMON_USER2_BASE, |
100 | .mmu_features = 0, |
101 | .icache_bsize = 32, /* minimum block size, fixed by */ |
102 | .dcache_bsize = 32, /* cache info init. */ |
103 | .num_pmcs = 0, |
104 | .pmc_type = PPC_PMC_DEFAULT, |
105 | .cpu_setup = NULL, |
106 | .cpu_restore = __restore_cpu_cpufeatures, |
107 | .machine_check_early = NULL, |
108 | .platform = NULL, |
109 | }; |
110 | |
111 | static void __init cpufeatures_setup_cpu(void) |
112 | { |
113 | set_cur_cpu_spec(&base_cpu_spec); |
114 | |
115 | cur_cpu_spec->pvr_mask = -1; |
116 | cur_cpu_spec->pvr_value = mfspr(SPRN_PVR); |
117 | |
118 | /* Initialize the base environment -- clear FSCR/HFSCR. */ |
119 | hv_mode = !!(mfmsr() & MSR_HV); |
120 | if (hv_mode) { |
121 | cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; |
122 | mtspr(SPRN_HFSCR, 0); |
123 | } |
124 | mtspr(SPRN_FSCR, 0); |
125 | mtspr(SPRN_PCR, PCR_MASK); |
126 | |
127 | /* |
128 | * LPCR does not get cleared, to match behaviour with secondaries |
129 | * in __restore_cpu_cpufeatures. Once the idle code is fixed, this |
130 | * could clear LPCR too. |
131 | */ |
132 | } |
133 | |
134 | static int __init feat_try_enable_unknown(struct dt_cpu_feature *f) |
135 | { |
136 | if (f->hv_support == HV_SUPPORT_NONE) { |
137 | } else if (f->hv_support & HV_SUPPORT_HFSCR) { |
138 | u64 hfscr = mfspr(SPRN_HFSCR); |
139 | hfscr |= 1UL << f->hfscr_bit_nr; |
140 | mtspr(SPRN_HFSCR, hfscr); |
141 | } else { |
142 | /* Does not have a known recipe */ |
143 | return 0; |
144 | } |
145 | |
146 | if (f->os_support == OS_SUPPORT_NONE) { |
147 | } else if (f->os_support & OS_SUPPORT_FSCR) { |
148 | u64 fscr = mfspr(SPRN_FSCR); |
149 | fscr |= 1UL << f->fscr_bit_nr; |
150 | mtspr(SPRN_FSCR, fscr); |
151 | } else { |
152 | /* Does not have a known recipe */ |
153 | return 0; |
154 | } |
155 | |
156 | if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { |
157 | uint32_t word = f->hwcap_bit_nr / 32; |
158 | uint32_t bit = f->hwcap_bit_nr % 32; |
159 | |
160 | if (word == 0) |
161 | cur_cpu_spec->cpu_user_features |= 1U << bit; |
162 | else if (word == 1) |
163 | cur_cpu_spec->cpu_user_features2 |= 1U << bit; |
164 | else |
165 | pr_err("%s could not advertise to user (no hwcap bits)\n" , f->name); |
166 | } |
167 | |
168 | return 1; |
169 | } |
170 | |
171 | static int __init feat_enable(struct dt_cpu_feature *f) |
172 | { |
173 | if (f->hv_support != HV_SUPPORT_NONE) { |
174 | if (f->hfscr_bit_nr != -1) { |
175 | u64 hfscr = mfspr(SPRN_HFSCR); |
176 | hfscr |= 1UL << f->hfscr_bit_nr; |
177 | mtspr(SPRN_HFSCR, hfscr); |
178 | } |
179 | } |
180 | |
181 | if (f->os_support != OS_SUPPORT_NONE) { |
182 | if (f->fscr_bit_nr != -1) { |
183 | u64 fscr = mfspr(SPRN_FSCR); |
184 | fscr |= 1UL << f->fscr_bit_nr; |
185 | mtspr(SPRN_FSCR, fscr); |
186 | } |
187 | } |
188 | |
189 | if ((f->usable_privilege & USABLE_PR) && (f->hwcap_bit_nr != -1)) { |
190 | uint32_t word = f->hwcap_bit_nr / 32; |
191 | uint32_t bit = f->hwcap_bit_nr % 32; |
192 | |
193 | if (word == 0) |
194 | cur_cpu_spec->cpu_user_features |= 1U << bit; |
195 | else if (word == 1) |
196 | cur_cpu_spec->cpu_user_features2 |= 1U << bit; |
197 | else |
198 | pr_err("CPU feature: %s could not advertise to user (no hwcap bits)\n" , f->name); |
199 | } |
200 | |
201 | return 1; |
202 | } |
203 | |
204 | static int __init feat_disable(struct dt_cpu_feature *f) |
205 | { |
206 | return 0; |
207 | } |
208 | |
209 | static int __init feat_enable_hv(struct dt_cpu_feature *f) |
210 | { |
211 | u64 lpcr; |
212 | |
213 | if (!hv_mode) { |
214 | pr_err("CPU feature hypervisor present in device tree but HV mode not enabled in the CPU. Ignoring.\n" ); |
215 | return 0; |
216 | } |
217 | |
218 | mtspr(SPRN_LPID, 0); |
219 | mtspr(SPRN_AMOR, ~0); |
220 | |
221 | lpcr = mfspr(SPRN_LPCR); |
222 | lpcr &= ~LPCR_LPES0; /* HV external interrupts */ |
223 | mtspr(SPRN_LPCR, lpcr); |
224 | |
225 | cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; |
226 | |
227 | return 1; |
228 | } |
229 | |
230 | static int __init feat_enable_le(struct dt_cpu_feature *f) |
231 | { |
232 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_TRUE_LE; |
233 | return 1; |
234 | } |
235 | |
236 | static int __init feat_enable_smt(struct dt_cpu_feature *f) |
237 | { |
238 | cur_cpu_spec->cpu_features |= CPU_FTR_SMT; |
239 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_SMT; |
240 | return 1; |
241 | } |
242 | |
243 | static int __init feat_enable_idle_nap(struct dt_cpu_feature *f) |
244 | { |
245 | u64 lpcr; |
246 | |
247 | /* Set PECE wakeup modes for ISA 207 */ |
248 | lpcr = mfspr(SPRN_LPCR); |
249 | lpcr |= LPCR_PECE0; |
250 | lpcr |= LPCR_PECE1; |
251 | lpcr |= LPCR_PECE2; |
252 | mtspr(SPRN_LPCR, lpcr); |
253 | |
254 | return 1; |
255 | } |
256 | |
257 | static int __init feat_enable_idle_stop(struct dt_cpu_feature *f) |
258 | { |
259 | u64 lpcr; |
260 | |
261 | /* Set PECE wakeup modes for ISAv3.0B */ |
262 | lpcr = mfspr(SPRN_LPCR); |
263 | lpcr |= LPCR_PECE0; |
264 | lpcr |= LPCR_PECE1; |
265 | lpcr |= LPCR_PECE2; |
266 | mtspr(SPRN_LPCR, lpcr); |
267 | |
268 | return 1; |
269 | } |
270 | |
271 | static int __init feat_enable_mmu_hash(struct dt_cpu_feature *f) |
272 | { |
273 | u64 lpcr; |
274 | |
275 | if (!IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) |
276 | return 0; |
277 | |
278 | lpcr = mfspr(SPRN_LPCR); |
279 | lpcr &= ~LPCR_ISL; |
280 | |
281 | /* VRMASD */ |
282 | lpcr |= LPCR_VPM0; |
283 | lpcr &= ~LPCR_VPM1; |
284 | lpcr |= 0x10UL << LPCR_VRMASD_SH; /* L=1 LP=00 */ |
285 | mtspr(SPRN_LPCR, lpcr); |
286 | |
287 | cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; |
288 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; |
289 | |
290 | return 1; |
291 | } |
292 | |
293 | static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f) |
294 | { |
295 | u64 lpcr; |
296 | |
297 | if (!IS_ENABLED(CONFIG_PPC_64S_HASH_MMU)) |
298 | return 0; |
299 | |
300 | lpcr = mfspr(SPRN_LPCR); |
301 | lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR); |
302 | mtspr(SPRN_LPCR, lpcr); |
303 | |
304 | cur_cpu_spec->mmu_features |= MMU_FTRS_HASH_BASE; |
305 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; |
306 | |
307 | return 1; |
308 | } |
309 | |
310 | |
311 | static int __init feat_enable_mmu_radix(struct dt_cpu_feature *f) |
312 | { |
313 | if (!IS_ENABLED(CONFIG_PPC_RADIX_MMU)) |
314 | return 0; |
315 | |
316 | cur_cpu_spec->mmu_features |= MMU_FTR_KERNEL_RO; |
317 | cur_cpu_spec->mmu_features |= MMU_FTR_TYPE_RADIX; |
318 | cur_cpu_spec->mmu_features |= MMU_FTR_GTSE; |
319 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_MMU; |
320 | |
321 | return 1; |
322 | } |
323 | |
324 | static int __init feat_enable_dscr(struct dt_cpu_feature *f) |
325 | { |
326 | u64 lpcr; |
327 | |
328 | /* |
329 | * Linux relies on FSCR[DSCR] being clear, so that we can take the |
330 | * facility unavailable interrupt and track the task's usage of DSCR. |
331 | * See facility_unavailable_exception(). |
332 | * Clear the bit here so that feat_enable() doesn't set it. |
333 | */ |
334 | f->fscr_bit_nr = -1; |
335 | |
336 | feat_enable(f); |
337 | |
338 | lpcr = mfspr(SPRN_LPCR); |
339 | lpcr &= ~LPCR_DPFD; |
340 | lpcr |= (4UL << LPCR_DPFD_SH); |
341 | mtspr(SPRN_LPCR, lpcr); |
342 | |
343 | return 1; |
344 | } |
345 | |
346 | static void __init hfscr_pmu_enable(void) |
347 | { |
348 | u64 hfscr = mfspr(SPRN_HFSCR); |
349 | hfscr |= PPC_BIT(60); |
350 | mtspr(SPRN_HFSCR, hfscr); |
351 | } |
352 | |
353 | static void init_pmu_power8(void) |
354 | { |
355 | if (hv_mode) { |
356 | mtspr(SPRN_MMCRC, 0); |
357 | mtspr(SPRN_MMCRH, 0); |
358 | } |
359 | |
360 | mtspr(SPRN_MMCRA, 0); |
361 | mtspr(SPRN_MMCR0, MMCR0_FC); |
362 | mtspr(SPRN_MMCR1, 0); |
363 | mtspr(SPRN_MMCR2, 0); |
364 | mtspr(SPRN_MMCRS, 0); |
365 | } |
366 | |
367 | static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) |
368 | { |
369 | cur_cpu_spec->platform = "power8" ; |
370 | cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; |
371 | |
372 | return 1; |
373 | } |
374 | |
375 | static int __init feat_enable_pmu_power8(struct dt_cpu_feature *f) |
376 | { |
377 | hfscr_pmu_enable(); |
378 | |
379 | init_pmu_power8(); |
380 | init_pmu_registers = init_pmu_power8; |
381 | |
382 | cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; |
383 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; |
384 | if (pvr_version_is(PVR_POWER8E)) |
385 | cur_cpu_spec->cpu_features |= CPU_FTR_PMAO_BUG; |
386 | |
387 | cur_cpu_spec->num_pmcs = 6; |
388 | cur_cpu_spec->pmc_type = PPC_PMC_IBM; |
389 | |
390 | return 1; |
391 | } |
392 | |
393 | static void init_pmu_power9(void) |
394 | { |
395 | if (hv_mode) |
396 | mtspr(SPRN_MMCRC, 0); |
397 | |
398 | mtspr(SPRN_MMCRA, 0); |
399 | mtspr(SPRN_MMCR0, MMCR0_FC); |
400 | mtspr(SPRN_MMCR1, 0); |
401 | mtspr(SPRN_MMCR2, 0); |
402 | } |
403 | |
404 | static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) |
405 | { |
406 | cur_cpu_spec->platform = "power9" ; |
407 | cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; |
408 | |
409 | return 1; |
410 | } |
411 | |
412 | static int __init feat_enable_pmu_power9(struct dt_cpu_feature *f) |
413 | { |
414 | hfscr_pmu_enable(); |
415 | |
416 | init_pmu_power9(); |
417 | init_pmu_registers = init_pmu_power9; |
418 | |
419 | cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; |
420 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; |
421 | |
422 | cur_cpu_spec->num_pmcs = 6; |
423 | cur_cpu_spec->pmc_type = PPC_PMC_IBM; |
424 | |
425 | return 1; |
426 | } |
427 | |
428 | static void init_pmu_power10(void) |
429 | { |
430 | init_pmu_power9(); |
431 | |
432 | mtspr(SPRN_MMCR3, 0); |
433 | mtspr(SPRN_MMCRA, MMCRA_BHRB_DISABLE); |
434 | mtspr(SPRN_MMCR0, MMCR0_FC | MMCR0_PMCCEXT); |
435 | } |
436 | |
437 | static int __init feat_enable_pmu_power10(struct dt_cpu_feature *f) |
438 | { |
439 | hfscr_pmu_enable(); |
440 | |
441 | init_pmu_power10(); |
442 | init_pmu_registers = init_pmu_power10; |
443 | |
444 | cur_cpu_spec->cpu_features |= CPU_FTR_MMCRA; |
445 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_PSERIES_PERFMON_COMPAT; |
446 | |
447 | cur_cpu_spec->num_pmcs = 6; |
448 | cur_cpu_spec->pmc_type = PPC_PMC_IBM; |
449 | |
450 | return 1; |
451 | } |
452 | |
453 | static int __init feat_enable_mce_power10(struct dt_cpu_feature *f) |
454 | { |
455 | cur_cpu_spec->platform = "power10" ; |
456 | cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; |
457 | |
458 | return 1; |
459 | } |
460 | |
461 | static int __init feat_enable_mce_power11(struct dt_cpu_feature *f) |
462 | { |
463 | cur_cpu_spec->platform = "power11" ; |
464 | cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p10; |
465 | |
466 | return 1; |
467 | } |
468 | |
469 | static int __init feat_enable_tm(struct dt_cpu_feature *f) |
470 | { |
471 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
472 | feat_enable(f); |
473 | cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_HTM_NOSC; |
474 | return 1; |
475 | #endif |
476 | return 0; |
477 | } |
478 | |
479 | static int __init feat_enable_fp(struct dt_cpu_feature *f) |
480 | { |
481 | feat_enable(f); |
482 | cur_cpu_spec->cpu_features &= ~CPU_FTR_FPU_UNAVAILABLE; |
483 | |
484 | return 1; |
485 | } |
486 | |
487 | static int __init feat_enable_vector(struct dt_cpu_feature *f) |
488 | { |
489 | #ifdef CONFIG_ALTIVEC |
490 | feat_enable(f); |
491 | cur_cpu_spec->cpu_features |= CPU_FTR_ALTIVEC; |
492 | cur_cpu_spec->cpu_features |= CPU_FTR_VMX_COPY; |
493 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_ALTIVEC; |
494 | |
495 | return 1; |
496 | #endif |
497 | return 0; |
498 | } |
499 | |
500 | static int __init feat_enable_vsx(struct dt_cpu_feature *f) |
501 | { |
502 | #ifdef CONFIG_VSX |
503 | feat_enable(f); |
504 | cur_cpu_spec->cpu_features |= CPU_FTR_VSX; |
505 | cur_cpu_spec->cpu_user_features |= PPC_FEATURE_HAS_VSX; |
506 | |
507 | return 1; |
508 | #endif |
509 | return 0; |
510 | } |
511 | |
512 | static int __init feat_enable_purr(struct dt_cpu_feature *f) |
513 | { |
514 | cur_cpu_spec->cpu_features |= CPU_FTR_PURR | CPU_FTR_SPURR; |
515 | |
516 | return 1; |
517 | } |
518 | |
519 | static int __init feat_enable_ebb(struct dt_cpu_feature *f) |
520 | { |
521 | /* |
522 | * PPC_FEATURE2_EBB is enabled in PMU init code because it has |
523 | * historically been related to the PMU facility. This may have |
524 | * to be decoupled if EBB becomes more generic. For now, follow |
525 | * existing convention. |
526 | */ |
527 | f->hwcap_bit_nr = -1; |
528 | feat_enable(f); |
529 | |
530 | return 1; |
531 | } |
532 | |
533 | static int __init feat_enable_dbell(struct dt_cpu_feature *f) |
534 | { |
535 | u64 lpcr; |
536 | |
537 | /* P9 has an HFSCR for privileged state */ |
538 | feat_enable(f); |
539 | |
540 | cur_cpu_spec->cpu_features |= CPU_FTR_DBELL; |
541 | |
542 | lpcr = mfspr(SPRN_LPCR); |
543 | lpcr |= LPCR_PECEDH; /* hyp doorbell wakeup */ |
544 | mtspr(SPRN_LPCR, lpcr); |
545 | |
546 | return 1; |
547 | } |
548 | |
549 | static int __init feat_enable_hvi(struct dt_cpu_feature *f) |
550 | { |
551 | u64 lpcr; |
552 | |
553 | /* |
554 | * POWER9 XIVE interrupts including in OPAL XICS compatibility |
555 | * are always delivered as hypervisor virtualization interrupts (HVI) |
556 | * rather than EE. |
557 | * |
558 | * However LPES0 is not set here, in the chance that an EE does get |
559 | * delivered to the host somehow, the EE handler would not expect it |
560 | * to be delivered in LPES0 mode (e.g., using SRR[01]). This could |
561 | * happen if there is a bug in interrupt controller code, or IC is |
562 | * misconfigured in systemsim. |
563 | */ |
564 | |
565 | lpcr = mfspr(SPRN_LPCR); |
566 | lpcr |= LPCR_HVICE; /* enable hvi interrupts */ |
567 | lpcr |= LPCR_HEIC; /* disable ee interrupts when MSR_HV */ |
568 | lpcr |= LPCR_PECE_HVEE; /* hvi can wake from stop */ |
569 | mtspr(SPRN_LPCR, lpcr); |
570 | |
571 | return 1; |
572 | } |
573 | |
574 | static int __init feat_enable_large_ci(struct dt_cpu_feature *f) |
575 | { |
576 | cur_cpu_spec->mmu_features |= MMU_FTR_CI_LARGE_PAGE; |
577 | |
578 | return 1; |
579 | } |
580 | |
581 | static int __init feat_enable_mma(struct dt_cpu_feature *f) |
582 | { |
583 | u64 pcr; |
584 | |
585 | feat_enable(f); |
586 | pcr = mfspr(SPRN_PCR); |
587 | pcr &= ~PCR_MMA_DIS; |
588 | mtspr(SPRN_PCR, pcr); |
589 | |
590 | return 1; |
591 | } |
592 | |
593 | struct dt_cpu_feature_match { |
594 | const char *name; |
595 | int (*enable)(struct dt_cpu_feature *f); |
596 | u64 cpu_ftr_bit_mask; |
597 | }; |
598 | |
599 | static struct dt_cpu_feature_match __initdata |
600 | dt_cpu_feature_match_table[] = { |
601 | {"hypervisor" , feat_enable_hv, 0}, |
602 | {"big-endian" , feat_enable, 0}, |
603 | {"little-endian" , feat_enable_le, CPU_FTR_REAL_LE}, |
604 | {"smt" , feat_enable_smt, 0}, |
605 | {"interrupt-facilities" , feat_enable, 0}, |
606 | {"system-call-vectored" , feat_enable, 0}, |
607 | {"timer-facilities" , feat_enable, 0}, |
608 | {"timer-facilities-v3" , feat_enable, 0}, |
609 | {"debug-facilities" , feat_enable, 0}, |
610 | {"come-from-address-register" , feat_enable, CPU_FTR_CFAR}, |
611 | {"branch-tracing" , feat_enable, 0}, |
612 | {"floating-point" , feat_enable_fp, 0}, |
613 | {"vector" , feat_enable_vector, 0}, |
614 | {"vector-scalar" , feat_enable_vsx, 0}, |
615 | {"vector-scalar-v3" , feat_enable, 0}, |
616 | {"decimal-floating-point" , feat_enable, 0}, |
617 | {"decimal-integer" , feat_enable, 0}, |
618 | {"quadword-load-store" , feat_enable, 0}, |
619 | {"vector-crypto" , feat_enable, 0}, |
620 | {"mmu-hash" , feat_enable_mmu_hash, 0}, |
621 | {"mmu-radix" , feat_enable_mmu_radix, 0}, |
622 | {"mmu-hash-v3" , feat_enable_mmu_hash_v3, 0}, |
623 | {"virtual-page-class-key-protection" , feat_enable, 0}, |
624 | {"transactional-memory" , feat_enable_tm, CPU_FTR_TM}, |
625 | {"transactional-memory-v3" , feat_enable_tm, 0}, |
626 | {"tm-suspend-hypervisor-assist" , feat_enable, CPU_FTR_P9_TM_HV_ASSIST}, |
627 | {"tm-suspend-xer-so-bug" , feat_enable, CPU_FTR_P9_TM_XER_SO_BUG}, |
628 | {"idle-nap" , feat_enable_idle_nap, 0}, |
629 | /* alignment-interrupt-dsisr ignored */ |
630 | {"idle-stop" , feat_enable_idle_stop, 0}, |
631 | {"machine-check-power8" , feat_enable_mce_power8, 0}, |
632 | {"performance-monitor-power8" , feat_enable_pmu_power8, 0}, |
633 | {"data-stream-control-register" , feat_enable_dscr, CPU_FTR_DSCR}, |
634 | {"event-based-branch" , feat_enable_ebb, 0}, |
635 | {"target-address-register" , feat_enable, 0}, |
636 | {"branch-history-rolling-buffer" , feat_enable, 0}, |
637 | {"control-register" , feat_enable, CPU_FTR_CTRL}, |
638 | {"processor-control-facility" , feat_enable_dbell, CPU_FTR_DBELL}, |
639 | {"processor-control-facility-v3" , feat_enable_dbell, CPU_FTR_DBELL}, |
640 | {"processor-utilization-of-resources-register" , feat_enable_purr, 0}, |
641 | {"no-execute" , feat_enable, 0}, |
642 | {"strong-access-ordering" , feat_enable, CPU_FTR_SAO}, |
643 | {"cache-inhibited-large-page" , feat_enable_large_ci, 0}, |
644 | {"coprocessor-icswx" , feat_enable, 0}, |
645 | {"hypervisor-virtualization-interrupt" , feat_enable_hvi, 0}, |
646 | {"program-priority-register" , feat_enable, CPU_FTR_HAS_PPR}, |
647 | {"wait" , feat_enable, 0}, |
648 | {"atomic-memory-operations" , feat_enable, 0}, |
649 | {"branch-v3" , feat_enable, 0}, |
650 | {"copy-paste" , feat_enable, 0}, |
651 | {"decimal-floating-point-v3" , feat_enable, 0}, |
652 | {"decimal-integer-v3" , feat_enable, 0}, |
653 | {"fixed-point-v3" , feat_enable, 0}, |
654 | {"floating-point-v3" , feat_enable, 0}, |
655 | {"group-start-register" , feat_enable, 0}, |
656 | {"pc-relative-addressing" , feat_enable, 0}, |
657 | {"machine-check-power9" , feat_enable_mce_power9, 0}, |
658 | {"machine-check-power10" , feat_enable_mce_power10, 0}, |
659 | {"machine-check-power11" , feat_enable_mce_power11, 0}, |
660 | {"performance-monitor-power9" , feat_enable_pmu_power9, 0}, |
661 | {"performance-monitor-power10" , feat_enable_pmu_power10, 0}, |
662 | {"performance-monitor-power11" , feat_enable_pmu_power10, 0}, |
663 | {"event-based-branch-v3" , feat_enable, 0}, |
664 | {"random-number-generator" , feat_enable, 0}, |
665 | {"system-call-vectored" , feat_disable, 0}, |
666 | {"trace-interrupt-v3" , feat_enable, 0}, |
667 | {"vector-v3" , feat_enable, 0}, |
668 | {"vector-binary128" , feat_enable, 0}, |
669 | {"vector-binary16" , feat_enable, 0}, |
670 | {"wait-v3" , feat_enable, 0}, |
671 | {"prefix-instructions" , feat_enable, 0}, |
672 | {"matrix-multiply-assist" , feat_enable_mma, 0}, |
673 | {"debug-facilities-v31" , feat_enable, CPU_FTR_DAWR1}, |
674 | }; |
675 | |
676 | static bool __initdata using_dt_cpu_ftrs; |
677 | static bool __initdata enable_unknown = true; |
678 | |
679 | static int __init dt_cpu_ftrs_parse(char *str) |
680 | { |
681 | if (!str) |
682 | return 0; |
683 | |
684 | if (!strcmp(str, "off" )) |
685 | using_dt_cpu_ftrs = false; |
686 | else if (!strcmp(str, "known" )) |
687 | enable_unknown = false; |
688 | else |
689 | return 1; |
690 | |
691 | return 0; |
692 | } |
693 | early_param("dt_cpu_ftrs" , dt_cpu_ftrs_parse); |
694 | |
695 | static void __init cpufeatures_setup_start(u32 isa) |
696 | { |
697 | pr_info("setup for ISA %d\n" , isa); |
698 | |
699 | if (isa >= ISA_V3_0B) { |
700 | cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_300; |
701 | cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_00; |
702 | } |
703 | |
704 | if (isa >= ISA_V3_1) { |
705 | cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31; |
706 | cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1; |
707 | } |
708 | } |
709 | |
710 | static bool __init cpufeatures_process_feature(struct dt_cpu_feature *f) |
711 | { |
712 | const struct dt_cpu_feature_match *m; |
713 | bool known = false; |
714 | int i; |
715 | |
716 | for (i = 0; i < ARRAY_SIZE(dt_cpu_feature_match_table); i++) { |
717 | m = &dt_cpu_feature_match_table[i]; |
718 | if (!strcmp(f->name, m->name)) { |
719 | known = true; |
720 | if (m->enable(f)) { |
721 | cur_cpu_spec->cpu_features |= m->cpu_ftr_bit_mask; |
722 | break; |
723 | } |
724 | |
725 | pr_info("not enabling: %s (disabled or unsupported by kernel)\n" , |
726 | f->name); |
727 | return false; |
728 | } |
729 | } |
730 | |
731 | if (!known && (!enable_unknown || !feat_try_enable_unknown(f))) { |
732 | pr_info("not enabling: %s (unknown and unsupported by kernel)\n" , |
733 | f->name); |
734 | return false; |
735 | } |
736 | |
737 | if (known) |
738 | pr_debug("enabling: %s\n" , f->name); |
739 | else |
740 | pr_debug("enabling: %s (unknown)\n" , f->name); |
741 | |
742 | return true; |
743 | } |
744 | |
745 | /* |
746 | * Handle POWER9 broadcast tlbie invalidation issue using |
747 | * cpu feature flag. |
748 | */ |
749 | static __init void update_tlbie_feature_flag(unsigned long pvr) |
750 | { |
751 | if (PVR_VER(pvr) == PVR_POWER9) { |
752 | /* |
753 | * Set the tlbie feature flag for anything below |
754 | * Nimbus DD 2.3 and Cumulus DD 1.3 |
755 | */ |
756 | if ((pvr & 0xe000) == 0) { |
757 | /* Nimbus */ |
758 | if ((pvr & 0xfff) < 0x203) |
759 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; |
760 | } else if ((pvr & 0xc000) == 0) { |
761 | /* Cumulus */ |
762 | if ((pvr & 0xfff) < 0x103) |
763 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; |
764 | } else { |
765 | WARN_ONCE(1, "Unknown PVR" ); |
766 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_STQ_BUG; |
767 | } |
768 | |
769 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_ERAT_BUG; |
770 | } |
771 | } |
772 | |
773 | static __init void cpufeatures_cpu_quirks(void) |
774 | { |
775 | unsigned long version = mfspr(SPRN_PVR); |
776 | |
777 | /* |
778 | * Not all quirks can be derived from the cpufeatures device tree. |
779 | */ |
780 | if ((version & 0xffffefff) == 0x004e0200) { |
781 | /* DD2.0 has no feature flag */ |
782 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; |
783 | cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); |
784 | } else if ((version & 0xffffefff) == 0x004e0201) { |
785 | cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; |
786 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_RADIX_PREFETCH_BUG; |
787 | cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); |
788 | } else if ((version & 0xffffefff) == 0x004e0202) { |
789 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; |
790 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; |
791 | cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; |
792 | cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); |
793 | } else if ((version & 0xffffefff) == 0x004e0203) { |
794 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST; |
795 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_XER_SO_BUG; |
796 | cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; |
797 | } else if ((version & 0xffff0000) == 0x004e0000) { |
798 | /* DD2.1 and up have DD2_1 */ |
799 | cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; |
800 | } |
801 | |
802 | if ((version & 0xffff0000) == 0x004e0000) { |
803 | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TIDR; |
804 | } |
805 | |
806 | update_tlbie_feature_flag(pvr: version); |
807 | } |
808 | |
809 | static void __init cpufeatures_setup_finished(void) |
810 | { |
811 | cpufeatures_cpu_quirks(); |
812 | |
813 | if (hv_mode && !(cur_cpu_spec->cpu_features & CPU_FTR_HVMODE)) { |
814 | pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n" ); |
815 | cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; |
816 | } |
817 | |
818 | /* Make sure powerpc_base_platform is non-NULL */ |
819 | powerpc_base_platform = cur_cpu_spec->platform; |
820 | |
821 | system_registers.lpcr = mfspr(SPRN_LPCR); |
822 | system_registers.hfscr = mfspr(SPRN_HFSCR); |
823 | system_registers.fscr = mfspr(SPRN_FSCR); |
824 | system_registers.pcr = mfspr(SPRN_PCR); |
825 | |
826 | pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n" , |
827 | cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); |
828 | } |
829 | |
830 | static int __init disabled_on_cmdline(void) |
831 | { |
832 | unsigned long root, chosen; |
833 | const char *p; |
834 | |
835 | root = of_get_flat_dt_root(); |
836 | chosen = of_get_flat_dt_subnode_by_name(node: root, uname: "chosen" ); |
837 | if (chosen == -FDT_ERR_NOTFOUND) |
838 | return false; |
839 | |
840 | p = of_get_flat_dt_prop(node: chosen, name: "bootargs" , NULL); |
841 | if (!p) |
842 | return false; |
843 | |
844 | if (strstr(p, "dt_cpu_ftrs=off" )) |
845 | return true; |
846 | |
847 | return false; |
848 | } |
849 | |
850 | static int __init fdt_find_cpu_features(unsigned long node, const char *uname, |
851 | int depth, void *data) |
852 | { |
853 | if (of_flat_dt_is_compatible(node, name: "ibm,powerpc-cpu-features" ) |
854 | && of_get_flat_dt_prop(node, name: "isa" , NULL)) |
855 | return 1; |
856 | |
857 | return 0; |
858 | } |
859 | |
860 | bool __init dt_cpu_ftrs_in_use(void) |
861 | { |
862 | return using_dt_cpu_ftrs; |
863 | } |
864 | |
865 | bool __init dt_cpu_ftrs_init(void *fdt) |
866 | { |
867 | using_dt_cpu_ftrs = false; |
868 | |
869 | /* Setup and verify the FDT, if it fails we just bail */ |
870 | if (!early_init_dt_verify(params: fdt)) |
871 | return false; |
872 | |
873 | if (!of_scan_flat_dt(it: fdt_find_cpu_features, NULL)) |
874 | return false; |
875 | |
876 | if (disabled_on_cmdline()) |
877 | return false; |
878 | |
879 | cpufeatures_setup_cpu(); |
880 | |
881 | using_dt_cpu_ftrs = true; |
882 | return true; |
883 | } |
884 | |
885 | static int nr_dt_cpu_features; |
886 | static struct dt_cpu_feature *dt_cpu_features; |
887 | |
888 | static int __init process_cpufeatures_node(unsigned long node, |
889 | const char *uname, int i) |
890 | { |
891 | const __be32 *prop; |
892 | struct dt_cpu_feature *f; |
893 | int len; |
894 | |
895 | f = &dt_cpu_features[i]; |
896 | |
897 | f->node = node; |
898 | |
899 | f->name = uname; |
900 | |
901 | prop = of_get_flat_dt_prop(node, name: "isa" , size: &len); |
902 | if (!prop) { |
903 | pr_warn("%s: missing isa property\n" , uname); |
904 | return 0; |
905 | } |
906 | f->isa = be32_to_cpup(p: prop); |
907 | |
908 | prop = of_get_flat_dt_prop(node, name: "usable-privilege" , size: &len); |
909 | if (!prop) { |
910 | pr_warn("%s: missing usable-privilege property" , uname); |
911 | return 0; |
912 | } |
913 | f->usable_privilege = be32_to_cpup(p: prop); |
914 | |
915 | prop = of_get_flat_dt_prop(node, name: "hv-support" , size: &len); |
916 | if (prop) |
917 | f->hv_support = be32_to_cpup(p: prop); |
918 | else |
919 | f->hv_support = HV_SUPPORT_NONE; |
920 | |
921 | prop = of_get_flat_dt_prop(node, name: "os-support" , size: &len); |
922 | if (prop) |
923 | f->os_support = be32_to_cpup(p: prop); |
924 | else |
925 | f->os_support = OS_SUPPORT_NONE; |
926 | |
927 | prop = of_get_flat_dt_prop(node, name: "hfscr-bit-nr" , size: &len); |
928 | if (prop) |
929 | f->hfscr_bit_nr = be32_to_cpup(p: prop); |
930 | else |
931 | f->hfscr_bit_nr = -1; |
932 | prop = of_get_flat_dt_prop(node, name: "fscr-bit-nr" , size: &len); |
933 | if (prop) |
934 | f->fscr_bit_nr = be32_to_cpup(p: prop); |
935 | else |
936 | f->fscr_bit_nr = -1; |
937 | prop = of_get_flat_dt_prop(node, name: "hwcap-bit-nr" , size: &len); |
938 | if (prop) |
939 | f->hwcap_bit_nr = be32_to_cpup(p: prop); |
940 | else |
941 | f->hwcap_bit_nr = -1; |
942 | |
943 | if (f->usable_privilege & USABLE_HV) { |
944 | if (!(mfmsr() & MSR_HV)) { |
945 | pr_warn("%s: HV feature passed to guest\n" , uname); |
946 | return 0; |
947 | } |
948 | |
949 | if (f->hv_support == HV_SUPPORT_NONE && f->hfscr_bit_nr != -1) { |
950 | pr_warn("%s: unwanted hfscr_bit_nr\n" , uname); |
951 | return 0; |
952 | } |
953 | |
954 | if (f->hv_support == HV_SUPPORT_HFSCR) { |
955 | if (f->hfscr_bit_nr == -1) { |
956 | pr_warn("%s: missing hfscr_bit_nr\n" , uname); |
957 | return 0; |
958 | } |
959 | } |
960 | } else { |
961 | if (f->hv_support != HV_SUPPORT_NONE || f->hfscr_bit_nr != -1) { |
962 | pr_warn("%s: unwanted hv_support/hfscr_bit_nr\n" , uname); |
963 | return 0; |
964 | } |
965 | } |
966 | |
967 | if (f->usable_privilege & USABLE_OS) { |
968 | if (f->os_support == OS_SUPPORT_NONE && f->fscr_bit_nr != -1) { |
969 | pr_warn("%s: unwanted fscr_bit_nr\n" , uname); |
970 | return 0; |
971 | } |
972 | |
973 | if (f->os_support == OS_SUPPORT_FSCR) { |
974 | if (f->fscr_bit_nr == -1) { |
975 | pr_warn("%s: missing fscr_bit_nr\n" , uname); |
976 | return 0; |
977 | } |
978 | } |
979 | } else { |
980 | if (f->os_support != OS_SUPPORT_NONE || f->fscr_bit_nr != -1) { |
981 | pr_warn("%s: unwanted os_support/fscr_bit_nr\n" , uname); |
982 | return 0; |
983 | } |
984 | } |
985 | |
986 | if (!(f->usable_privilege & USABLE_PR)) { |
987 | if (f->hwcap_bit_nr != -1) { |
988 | pr_warn("%s: unwanted hwcap_bit_nr\n" , uname); |
989 | return 0; |
990 | } |
991 | } |
992 | |
993 | /* Do all the independent features in the first pass */ |
994 | if (!of_get_flat_dt_prop(node, name: "dependencies" , size: &len)) { |
995 | if (cpufeatures_process_feature(f)) |
996 | f->enabled = 1; |
997 | else |
998 | f->disabled = 1; |
999 | } |
1000 | |
1001 | return 0; |
1002 | } |
1003 | |
1004 | static void __init cpufeatures_deps_enable(struct dt_cpu_feature *f) |
1005 | { |
1006 | const __be32 *prop; |
1007 | int len; |
1008 | int nr_deps; |
1009 | int i; |
1010 | |
1011 | if (f->enabled || f->disabled) |
1012 | return; |
1013 | |
1014 | prop = of_get_flat_dt_prop(node: f->node, name: "dependencies" , size: &len); |
1015 | if (!prop) { |
1016 | pr_warn("%s: missing dependencies property" , f->name); |
1017 | return; |
1018 | } |
1019 | |
1020 | nr_deps = len / sizeof(int); |
1021 | |
1022 | for (i = 0; i < nr_deps; i++) { |
1023 | unsigned long phandle = be32_to_cpu(prop[i]); |
1024 | int j; |
1025 | |
1026 | for (j = 0; j < nr_dt_cpu_features; j++) { |
1027 | struct dt_cpu_feature *d = &dt_cpu_features[j]; |
1028 | |
1029 | if (of_get_flat_dt_phandle(node: d->node) == phandle) { |
1030 | cpufeatures_deps_enable(f: d); |
1031 | if (d->disabled) { |
1032 | f->disabled = 1; |
1033 | return; |
1034 | } |
1035 | } |
1036 | } |
1037 | } |
1038 | |
1039 | if (cpufeatures_process_feature(f)) |
1040 | f->enabled = 1; |
1041 | else |
1042 | f->disabled = 1; |
1043 | } |
1044 | |
1045 | static int __init scan_cpufeatures_subnodes(unsigned long node, |
1046 | const char *uname, |
1047 | void *data) |
1048 | { |
1049 | int *count = data; |
1050 | |
1051 | process_cpufeatures_node(node, uname, i: *count); |
1052 | |
1053 | (*count)++; |
1054 | |
1055 | return 0; |
1056 | } |
1057 | |
1058 | static int __init count_cpufeatures_subnodes(unsigned long node, |
1059 | const char *uname, |
1060 | void *data) |
1061 | { |
1062 | int *count = data; |
1063 | |
1064 | (*count)++; |
1065 | |
1066 | return 0; |
1067 | } |
1068 | |
1069 | static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char |
1070 | *uname, int depth, void *data) |
1071 | { |
1072 | const __be32 *prop; |
1073 | int count, i; |
1074 | u32 isa; |
1075 | |
1076 | /* We are scanning "ibm,powerpc-cpu-features" nodes only */ |
1077 | if (!of_flat_dt_is_compatible(node, name: "ibm,powerpc-cpu-features" )) |
1078 | return 0; |
1079 | |
1080 | prop = of_get_flat_dt_prop(node, name: "isa" , NULL); |
1081 | if (!prop) |
1082 | /* We checked before, "can't happen" */ |
1083 | return 0; |
1084 | |
1085 | isa = be32_to_cpup(p: prop); |
1086 | |
1087 | /* Count and allocate space for cpu features */ |
1088 | of_scan_flat_dt_subnodes(node, it: count_cpufeatures_subnodes, |
1089 | data: &nr_dt_cpu_features); |
1090 | dt_cpu_features = memblock_alloc(size: sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); |
1091 | if (!dt_cpu_features) |
1092 | panic(fmt: "%s: Failed to allocate %zu bytes align=0x%lx\n" , |
1093 | __func__, |
1094 | sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, |
1095 | PAGE_SIZE); |
1096 | |
1097 | cpufeatures_setup_start(isa); |
1098 | |
1099 | /* Scan nodes into dt_cpu_features and enable those without deps */ |
1100 | count = 0; |
1101 | of_scan_flat_dt_subnodes(node, it: scan_cpufeatures_subnodes, data: &count); |
1102 | |
1103 | /* Recursive enable remaining features with dependencies */ |
1104 | for (i = 0; i < nr_dt_cpu_features; i++) { |
1105 | struct dt_cpu_feature *f = &dt_cpu_features[i]; |
1106 | |
1107 | cpufeatures_deps_enable(f); |
1108 | } |
1109 | |
1110 | prop = of_get_flat_dt_prop(node, name: "display-name" , NULL); |
1111 | if (prop && strlen((char *)prop) != 0) { |
1112 | strscpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); |
1113 | cur_cpu_spec->cpu_name = dt_cpu_name; |
1114 | } |
1115 | |
1116 | cpufeatures_setup_finished(); |
1117 | |
1118 | memblock_free(ptr: dt_cpu_features, |
1119 | size: sizeof(struct dt_cpu_feature) * nr_dt_cpu_features); |
1120 | |
1121 | return 0; |
1122 | } |
1123 | |
1124 | void __init dt_cpu_ftrs_scan(void) |
1125 | { |
1126 | if (!using_dt_cpu_ftrs) |
1127 | return; |
1128 | |
1129 | of_scan_flat_dt(it: dt_cpu_ftrs_scan_callback, NULL); |
1130 | } |
1131 | |