1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Contains CPU feature definitions |
4 | * |
5 | * Copyright (C) 2015 ARM Ltd. |
6 | * |
7 | * A note for the weary kernel hacker: the code here is confusing and hard to |
8 | * follow! That's partly because it's solving a nasty problem, but also because |
9 | * there's a little bit of over-abstraction that tends to obscure what's going |
10 | * on behind a maze of helper functions and macros. |
11 | * |
12 | * The basic problem is that hardware folks have started gluing together CPUs |
13 | * with distinct architectural features; in some cases even creating SoCs where |
14 | * user-visible instructions are available only on a subset of the available |
15 | * cores. We try to address this by snapshotting the feature registers of the |
16 | * boot CPU and comparing these with the feature registers of each secondary |
17 | * CPU when bringing them up. If there is a mismatch, then we update the |
18 | * snapshot state to indicate the lowest-common denominator of the feature, |
19 | * known as the "safe" value. This snapshot state can be queried to view the |
20 | * "sanitised" value of a feature register. |
21 | * |
22 | * The sanitised register values are used to decide which capabilities we |
23 | * have in the system. These may be in the form of traditional "hwcaps" |
24 | * advertised to userspace or internal "cpucaps" which are used to configure |
25 | * things like alternative patching and static keys. While a feature mismatch |
26 | * may result in a TAINT_CPU_OUT_OF_SPEC kernel taint, a capability mismatch |
27 | * may prevent a CPU from being onlined at all. |
28 | * |
29 | * Some implementation details worth remembering: |
30 | * |
31 | * - Mismatched features are *always* sanitised to a "safe" value, which |
32 | * usually indicates that the feature is not supported. |
33 | * |
34 | * - A mismatched feature marked with FTR_STRICT will cause a "SANITY CHECK" |
35 | * warning when onlining an offending CPU and the kernel will be tainted |
36 | * with TAINT_CPU_OUT_OF_SPEC. |
37 | * |
38 | * - Features marked as FTR_VISIBLE have their sanitised value visible to |
39 | * userspace. FTR_VISIBLE features in registers that are only visible |
40 | * to EL0 by trapping *must* have a corresponding HWCAP so that late |
41 | * onlining of CPUs cannot lead to features disappearing at runtime. |
42 | * |
43 | * - A "feature" is typically a 4-bit register field. A "capability" is the |
44 | * high-level description derived from the sanitised field value. |
45 | * |
46 | * - Read the Arm ARM (DDI 0487F.a) section D13.1.3 ("Principles of the ID |
47 | * scheme for fields in ID registers") to understand when feature fields |
48 | * may be signed or unsigned (FTR_SIGNED and FTR_UNSIGNED accordingly). |
49 | * |
50 | * - KVM exposes its own view of the feature registers to guest operating |
51 | * systems regardless of FTR_VISIBLE. This is typically driven from the |
52 | * sanitised register values to allow virtual CPUs to be migrated between |
53 | * arbitrary physical CPUs, but some features not present on the host are |
54 | * also advertised and emulated. Look at sys_reg_descs[] for the gory |
55 | * details. |
56 | * |
57 | * - If the arm64_ftr_bits[] for a register has a missing field, then this |
58 | * field is treated as STRICT RES0, including for read_sanitised_ftr_reg(). |
59 | * This is stronger than FTR_HIDDEN and can be used to hide features from |
60 | * KVM guests. |
61 | */ |
62 | |
63 | #define pr_fmt(fmt) "CPU features: " fmt |
64 | |
65 | #include <linux/bsearch.h> |
66 | #include <linux/cpumask.h> |
67 | #include <linux/crash_dump.h> |
68 | #include <linux/kstrtox.h> |
69 | #include <linux/sort.h> |
70 | #include <linux/stop_machine.h> |
71 | #include <linux/sysfs.h> |
72 | #include <linux/types.h> |
73 | #include <linux/minmax.h> |
74 | #include <linux/mm.h> |
75 | #include <linux/cpu.h> |
76 | #include <linux/kasan.h> |
77 | #include <linux/percpu.h> |
78 | |
79 | #include <asm/cpu.h> |
80 | #include <asm/cpufeature.h> |
81 | #include <asm/cpu_ops.h> |
82 | #include <asm/fpsimd.h> |
83 | #include <asm/hwcap.h> |
84 | #include <asm/insn.h> |
85 | #include <asm/kvm_host.h> |
86 | #include <asm/mmu_context.h> |
87 | #include <asm/mte.h> |
88 | #include <asm/processor.h> |
89 | #include <asm/smp.h> |
90 | #include <asm/sysreg.h> |
91 | #include <asm/traps.h> |
92 | #include <asm/vectors.h> |
93 | #include <asm/virt.h> |
94 | |
95 | /* Kernel representation of AT_HWCAP and AT_HWCAP2 */ |
96 | static DECLARE_BITMAP(elf_hwcap, MAX_CPU_FEATURES) __read_mostly; |
97 | |
98 | #ifdef CONFIG_COMPAT |
99 | #define COMPAT_ELF_HWCAP_DEFAULT \ |
100 | (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\ |
101 | COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\ |
102 | COMPAT_HWCAP_TLS|COMPAT_HWCAP_IDIV|\ |
103 | COMPAT_HWCAP_LPAE) |
104 | unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT; |
105 | unsigned int compat_elf_hwcap2 __read_mostly; |
106 | #endif |
107 | |
108 | DECLARE_BITMAP(system_cpucaps, ARM64_NCAPS); |
109 | EXPORT_SYMBOL(system_cpucaps); |
110 | static struct arm64_cpu_capabilities const __ro_after_init *cpucap_ptrs[ARM64_NCAPS]; |
111 | |
112 | DECLARE_BITMAP(boot_cpucaps, ARM64_NCAPS); |
113 | |
114 | bool arm64_use_ng_mappings = false; |
115 | EXPORT_SYMBOL(arm64_use_ng_mappings); |
116 | |
117 | DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors; |
118 | |
119 | /* |
120 | * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs |
121 | * support it? |
122 | */ |
123 | static bool __read_mostly allow_mismatched_32bit_el0; |
124 | |
125 | /* |
126 | * Static branch enabled only if allow_mismatched_32bit_el0 is set and we have |
127 | * seen at least one CPU capable of 32-bit EL0. |
128 | */ |
129 | DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); |
130 | |
131 | /* |
132 | * Mask of CPUs supporting 32-bit EL0. |
133 | * Only valid if arm64_mismatched_32bit_el0 is enabled. |
134 | */ |
135 | static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly; |
136 | |
137 | void dump_cpu_features(void) |
138 | { |
139 | /* file-wide pr_fmt adds "CPU features: " prefix */ |
140 | pr_emerg("0x%*pb\n" , ARM64_NCAPS, &system_cpucaps); |
141 | } |
142 | |
143 | #define __ARM64_MAX_POSITIVE(reg, field) \ |
144 | ((reg##_##field##_SIGNED ? \ |
145 | BIT(reg##_##field##_WIDTH - 1) : \ |
146 | BIT(reg##_##field##_WIDTH)) - 1) |
147 | |
148 | #define __ARM64_MIN_NEGATIVE(reg, field) BIT(reg##_##field##_WIDTH - 1) |
149 | |
150 | #define __ARM64_CPUID_FIELDS(reg, field, min_value, max_value) \ |
151 | .sys_reg = SYS_##reg, \ |
152 | .field_pos = reg##_##field##_SHIFT, \ |
153 | .field_width = reg##_##field##_WIDTH, \ |
154 | .sign = reg##_##field##_SIGNED, \ |
155 | .min_field_value = min_value, \ |
156 | .max_field_value = max_value, |
157 | |
158 | /* |
159 | * ARM64_CPUID_FIELDS() encodes a field with a range from min_value to |
160 | * an implicit maximum that depends on the sign-ess of the field. |
161 | * |
162 | * An unsigned field will be capped at all ones, while a signed field |
163 | * will be limited to the positive half only. |
164 | */ |
165 | #define ARM64_CPUID_FIELDS(reg, field, min_value) \ |
166 | __ARM64_CPUID_FIELDS(reg, field, \ |
167 | SYS_FIELD_VALUE(reg, field, min_value), \ |
168 | __ARM64_MAX_POSITIVE(reg, field)) |
169 | |
170 | /* |
171 | * ARM64_CPUID_FIELDS_NEG() encodes a field with a range from an |
172 | * implicit minimal value to max_value. This should be used when |
173 | * matching a non-implemented property. |
174 | */ |
175 | #define ARM64_CPUID_FIELDS_NEG(reg, field, max_value) \ |
176 | __ARM64_CPUID_FIELDS(reg, field, \ |
177 | __ARM64_MIN_NEGATIVE(reg, field), \ |
178 | SYS_FIELD_VALUE(reg, field, max_value)) |
179 | |
180 | #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \ |
181 | { \ |
182 | .sign = SIGNED, \ |
183 | .visible = VISIBLE, \ |
184 | .strict = STRICT, \ |
185 | .type = TYPE, \ |
186 | .shift = SHIFT, \ |
187 | .width = WIDTH, \ |
188 | .safe_val = SAFE_VAL, \ |
189 | } |
190 | |
191 | /* Define a feature with unsigned values */ |
192 | #define ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \ |
193 | __ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) |
194 | |
195 | /* Define a feature with a signed value */ |
196 | #define S_ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \ |
197 | __ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) |
198 | |
199 | #define ARM64_FTR_END \ |
200 | { \ |
201 | .width = 0, \ |
202 | } |
203 | |
204 | static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap); |
205 | |
206 | static bool __system_matches_cap(unsigned int n); |
207 | |
208 | /* |
209 | * NOTE: Any changes to the visibility of features should be kept in |
210 | * sync with the documentation of the CPU feature register ABI. |
211 | */ |
212 | static const struct arm64_ftr_bits ftr_id_aa64isar0[] = { |
213 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_RNDR_SHIFT, 4, 0), |
214 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_TLB_SHIFT, 4, 0), |
215 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_TS_SHIFT, 4, 0), |
216 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_FHM_SHIFT, 4, 0), |
217 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_DP_SHIFT, 4, 0), |
218 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SM4_SHIFT, 4, 0), |
219 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SM3_SHIFT, 4, 0), |
220 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SHA3_SHIFT, 4, 0), |
221 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_RDM_SHIFT, 4, 0), |
222 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_ATOMIC_SHIFT, 4, 0), |
223 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_CRC32_SHIFT, 4, 0), |
224 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SHA2_SHIFT, 4, 0), |
225 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SHA1_SHIFT, 4, 0), |
226 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_AES_SHIFT, 4, 0), |
227 | ARM64_FTR_END, |
228 | }; |
229 | |
230 | static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { |
231 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_I8MM_SHIFT, 4, 0), |
232 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_DGH_SHIFT, 4, 0), |
233 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_BF16_SHIFT, 4, 0), |
234 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_SPECRES_SHIFT, 4, 0), |
235 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_SB_SHIFT, 4, 0), |
236 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_FRINTTS_SHIFT, 4, 0), |
237 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), |
238 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_GPI_SHIFT, 4, 0), |
239 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), |
240 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_GPA_SHIFT, 4, 0), |
241 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_LRCPC_SHIFT, 4, 0), |
242 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_FCMA_SHIFT, 4, 0), |
243 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_JSCVT_SHIFT, 4, 0), |
244 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), |
245 | FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_EL1_API_SHIFT, 4, 0), |
246 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), |
247 | FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_EL1_APA_SHIFT, 4, 0), |
248 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_DPB_SHIFT, 4, 0), |
249 | ARM64_FTR_END, |
250 | }; |
251 | |
252 | static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { |
253 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_LUT_SHIFT, 4, 0), |
254 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0), |
255 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0), |
256 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CLRBHB_SHIFT, 4, 0), |
257 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0), |
258 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_MOPS_SHIFT, 4, 0), |
259 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), |
260 | FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0), |
261 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), |
262 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_GPA3_SHIFT, 4, 0), |
263 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, 0), |
264 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, 0), |
265 | ARM64_FTR_END, |
266 | }; |
267 | |
268 | static const struct arm64_ftr_bits ftr_id_aa64isar3[] = { |
269 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_FAMINMAX_SHIFT, 4, 0), |
270 | ARM64_FTR_END, |
271 | }; |
272 | |
273 | static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { |
274 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV3_SHIFT, 4, 0), |
275 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV2_SHIFT, 4, 0), |
276 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_DIT_SHIFT, 4, 0), |
277 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AMU_SHIFT, 4, 0), |
278 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_MPAM_SHIFT, 4, 0), |
279 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SEL2_SHIFT, 4, 0), |
280 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
281 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SVE_SHIFT, 4, 0), |
282 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_RAS_SHIFT, 4, 0), |
283 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_GIC_SHIFT, 4, 0), |
284 | S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AdvSIMD_SHIFT, 4, ID_AA64PFR0_EL1_AdvSIMD_NI), |
285 | S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_FP_SHIFT, 4, ID_AA64PFR0_EL1_FP_NI), |
286 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL3_SHIFT, 4, 0), |
287 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL2_SHIFT, 4, 0), |
288 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL1_SHIFT, 4, ID_AA64PFR0_EL1_ELx_64BIT_ONLY), |
289 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL0_SHIFT, 4, ID_AA64PFR0_EL1_ELx_64BIT_ONLY), |
290 | ARM64_FTR_END, |
291 | }; |
292 | |
293 | static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = { |
294 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
295 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SME_SHIFT, 4, 0), |
296 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, 4, 0), |
297 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_RAS_frac_SHIFT, 4, 0), |
298 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_MTE), |
299 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_SHIFT, 4, ID_AA64PFR1_EL1_MTE_NI), |
300 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SSBS_SHIFT, 4, ID_AA64PFR1_EL1_SSBS_NI), |
301 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_BTI), |
302 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_BT_SHIFT, 4, 0), |
303 | ARM64_FTR_END, |
304 | }; |
305 | |
306 | static const struct arm64_ftr_bits ftr_id_aa64pfr2[] = { |
307 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_FPMR_SHIFT, 4, 0), |
308 | ARM64_FTR_END, |
309 | }; |
310 | |
311 | static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { |
312 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
313 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F64MM_SHIFT, 4, 0), |
314 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
315 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F32MM_SHIFT, 4, 0), |
316 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
317 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_I8MM_SHIFT, 4, 0), |
318 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
319 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SM4_SHIFT, 4, 0), |
320 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
321 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SHA3_SHIFT, 4, 0), |
322 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
323 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_B16B16_SHIFT, 4, 0), |
324 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
325 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_BF16_SHIFT, 4, 0), |
326 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
327 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_BitPerm_SHIFT, 4, 0), |
328 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
329 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_AES_SHIFT, 4, 0), |
330 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), |
331 | FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, 0), |
332 | ARM64_FTR_END, |
333 | }; |
334 | |
335 | static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { |
336 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
337 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0), |
338 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
339 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUTv2_SHIFT, 1, 0), |
340 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
341 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SMEver_SHIFT, 4, 0), |
342 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
343 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I16I64_SHIFT, 4, 0), |
344 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
345 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F64F64_SHIFT, 1, 0), |
346 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
347 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I16I32_SHIFT, 4, 0), |
348 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
349 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_B16B16_SHIFT, 1, 0), |
350 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
351 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F16F16_SHIFT, 1, 0), |
352 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
353 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F8F16_SHIFT, 1, 0), |
354 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
355 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F8F32_SHIFT, 1, 0), |
356 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
357 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I8I32_SHIFT, 4, 0), |
358 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
359 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F16F32_SHIFT, 1, 0), |
360 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
361 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_B16F32_SHIFT, 1, 0), |
362 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
363 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_BI32I32_SHIFT, 1, 0), |
364 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
365 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F32F32_SHIFT, 1, 0), |
366 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
367 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8FMA_SHIFT, 1, 0), |
368 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
369 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8DP4_SHIFT, 1, 0), |
370 | ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), |
371 | FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8DP2_SHIFT, 1, 0), |
372 | ARM64_FTR_END, |
373 | }; |
374 | |
375 | static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = { |
376 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8CVT_SHIFT, 1, 0), |
377 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8FMA_SHIFT, 1, 0), |
378 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP4_SHIFT, 1, 0), |
379 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0), |
380 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0), |
381 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0), |
382 | ARM64_FTR_END, |
383 | }; |
384 | |
385 | static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { |
386 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, 0), |
387 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_FGT_SHIFT, 4, 0), |
388 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_EXS_SHIFT, 4, 0), |
389 | /* |
390 | * Page size not being supported at Stage-2 is not fatal. You |
391 | * just give up KVM if PAGE_SIZE isn't supported there. Go fix |
392 | * your favourite nesting hypervisor. |
393 | * |
394 | * There is a small corner case where the hypervisor explicitly |
395 | * advertises a given granule size at Stage-2 (value 2) on some |
396 | * vCPUs, and uses the fallback to Stage-1 (value 0) for other |
397 | * vCPUs. Although this is not forbidden by the architecture, it |
398 | * indicates that the hypervisor is being silly (or buggy). |
399 | * |
400 | * We make no effort to cope with this and pretend that if these |
401 | * fields are inconsistent across vCPUs, then it isn't worth |
402 | * trying to bring KVM up. |
403 | */ |
404 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT, 4, 1), |
405 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT, 4, 1), |
406 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT, 4, 1), |
407 | /* |
408 | * We already refuse to boot CPUs that don't support our configured |
409 | * page size, so we can only detect mismatches for a page size other |
410 | * than the one we're currently using. Unfortunately, SoCs like this |
411 | * exist in the wild so, even though we don't like it, we'll have to go |
412 | * along with it and treat them as non-strict. |
413 | */ |
414 | S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN4_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN4_NI), |
415 | S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN64_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN64_NI), |
416 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN16_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN16_NI), |
417 | |
418 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGENDEL0_SHIFT, 4, 0), |
419 | /* Linux shouldn't care about secure memory */ |
420 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_SNSMEM_SHIFT, 4, 0), |
421 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGEND_SHIFT, 4, 0), |
422 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT, 4, 0), |
423 | /* |
424 | * Differing PARange is fine as long as all peripherals and memory are mapped |
425 | * within the minimum PARange of all CPUs |
426 | */ |
427 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_PARANGE_SHIFT, 4, 0), |
428 | ARM64_FTR_END, |
429 | }; |
430 | |
431 | static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = { |
432 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TIDCP1_SHIFT, 4, 0), |
433 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, 0), |
434 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HCX_SHIFT, 4, 0), |
435 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_ETS_SHIFT, 4, 0), |
436 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TWED_SHIFT, 4, 0), |
437 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_XNX_SHIFT, 4, 0), |
438 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64MMFR1_EL1_SpecSEI_SHIFT, 4, 0), |
439 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_PAN_SHIFT, 4, 0), |
440 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_LO_SHIFT, 4, 0), |
441 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HPDS_SHIFT, 4, 0), |
442 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VH_SHIFT, 4, 0), |
443 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VMIDBits_SHIFT, 4, 0), |
444 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, 4, 0), |
445 | ARM64_FTR_END, |
446 | }; |
447 | |
448 | static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = { |
449 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_E0PD_SHIFT, 4, 0), |
450 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_EVT_SHIFT, 4, 0), |
451 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_BBM_SHIFT, 4, 0), |
452 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_TTL_SHIFT, 4, 0), |
453 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_FWB_SHIFT, 4, 0), |
454 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IDS_SHIFT, 4, 0), |
455 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_AT_SHIFT, 4, 0), |
456 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_ST_SHIFT, 4, 0), |
457 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_NV_SHIFT, 4, 0), |
458 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CCIDX_SHIFT, 4, 0), |
459 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_VARange_SHIFT, 4, 0), |
460 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IESB_SHIFT, 4, 0), |
461 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_LSM_SHIFT, 4, 0), |
462 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_UAO_SHIFT, 4, 0), |
463 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CnP_SHIFT, 4, 0), |
464 | ARM64_FTR_END, |
465 | }; |
466 | |
467 | static const struct arm64_ftr_bits ftr_id_aa64mmfr3[] = { |
468 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1PIE_SHIFT, 4, 0), |
469 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_TCRX_SHIFT, 4, 0), |
470 | ARM64_FTR_END, |
471 | }; |
472 | |
473 | static const struct arm64_ftr_bits ftr_id_aa64mmfr4[] = { |
474 | S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR4_EL1_E2H0_SHIFT, 4, 0), |
475 | ARM64_FTR_END, |
476 | }; |
477 | |
478 | static const struct arm64_ftr_bits ftr_ctr[] = { |
479 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */ |
480 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_DIC_SHIFT, 1, 1), |
481 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_IDC_SHIFT, 1, 1), |
482 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_EL0_CWG_SHIFT, 4, 0), |
483 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_EL0_ERG_SHIFT, 4, 0), |
484 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_DminLine_SHIFT, 4, 1), |
485 | /* |
486 | * Linux can handle differing I-cache policies. Userspace JITs will |
487 | * make use of *minLine. |
488 | * If we have differing I-cache policies, report it as the weakest - VIPT. |
489 | */ |
490 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, CTR_EL0_L1Ip_SHIFT, 2, CTR_EL0_L1Ip_VIPT), /* L1Ip */ |
491 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_IminLine_SHIFT, 4, 0), |
492 | ARM64_FTR_END, |
493 | }; |
494 | |
495 | static struct arm64_ftr_override __ro_after_init no_override = { }; |
496 | |
497 | struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = { |
498 | .name = "SYS_CTR_EL0" , |
499 | .ftr_bits = ftr_ctr, |
500 | .override = &no_override, |
501 | }; |
502 | |
503 | static const struct arm64_ftr_bits ftr_id_mmfr0[] = { |
504 | S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_InnerShr_SHIFT, 4, 0xf), |
505 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_FCSE_SHIFT, 4, 0), |
506 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_AuxReg_SHIFT, 4, 0), |
507 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_TCM_SHIFT, 4, 0), |
508 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_ShareLvl_SHIFT, 4, 0), |
509 | S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_OuterShr_SHIFT, 4, 0xf), |
510 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_PMSA_SHIFT, 4, 0), |
511 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_VMSA_SHIFT, 4, 0), |
512 | ARM64_FTR_END, |
513 | }; |
514 | |
515 | static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { |
516 | S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_DoubleLock_SHIFT, 4, 0), |
517 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_PMSVer_SHIFT, 4, 0), |
518 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_CTX_CMPs_SHIFT, 4, 0), |
519 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_WRPs_SHIFT, 4, 0), |
520 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_BRPs_SHIFT, 4, 0), |
521 | /* |
522 | * We can instantiate multiple PMU instances with different levels |
523 | * of support. |
524 | */ |
525 | S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_EL1_PMUVer_SHIFT, 4, 0), |
526 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_EL1_DebugVer_SHIFT, 4, 0x6), |
527 | ARM64_FTR_END, |
528 | }; |
529 | |
530 | static const struct arm64_ftr_bits ftr_mvfr0[] = { |
531 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPRound_SHIFT, 4, 0), |
532 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPShVec_SHIFT, 4, 0), |
533 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSqrt_SHIFT, 4, 0), |
534 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDivide_SHIFT, 4, 0), |
535 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPTrap_SHIFT, 4, 0), |
536 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDP_SHIFT, 4, 0), |
537 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSP_SHIFT, 4, 0), |
538 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_SIMDReg_SHIFT, 4, 0), |
539 | ARM64_FTR_END, |
540 | }; |
541 | |
542 | static const struct arm64_ftr_bits ftr_mvfr1[] = { |
543 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDFMAC_SHIFT, 4, 0), |
544 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPHP_SHIFT, 4, 0), |
545 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDHP_SHIFT, 4, 0), |
546 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDSP_SHIFT, 4, 0), |
547 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDInt_SHIFT, 4, 0), |
548 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDLS_SHIFT, 4, 0), |
549 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPDNaN_SHIFT, 4, 0), |
550 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPFtZ_SHIFT, 4, 0), |
551 | ARM64_FTR_END, |
552 | }; |
553 | |
554 | static const struct arm64_ftr_bits ftr_mvfr2[] = { |
555 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_FPMisc_SHIFT, 4, 0), |
556 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_SIMDMisc_SHIFT, 4, 0), |
557 | ARM64_FTR_END, |
558 | }; |
559 | |
560 | static const struct arm64_ftr_bits ftr_dczid[] = { |
561 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, DCZID_EL0_DZP_SHIFT, 1, 1), |
562 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, DCZID_EL0_BS_SHIFT, 4, 0), |
563 | ARM64_FTR_END, |
564 | }; |
565 | |
566 | static const struct arm64_ftr_bits ftr_gmid[] = { |
567 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, GMID_EL1_BS_SHIFT, 4, 0), |
568 | ARM64_FTR_END, |
569 | }; |
570 | |
571 | static const struct arm64_ftr_bits ftr_id_isar0[] = { |
572 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Divide_SHIFT, 4, 0), |
573 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Debug_SHIFT, 4, 0), |
574 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Coproc_SHIFT, 4, 0), |
575 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_CmpBranch_SHIFT, 4, 0), |
576 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitField_SHIFT, 4, 0), |
577 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitCount_SHIFT, 4, 0), |
578 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Swap_SHIFT, 4, 0), |
579 | ARM64_FTR_END, |
580 | }; |
581 | |
582 | static const struct arm64_ftr_bits ftr_id_isar5[] = { |
583 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_RDM_SHIFT, 4, 0), |
584 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_CRC32_SHIFT, 4, 0), |
585 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA2_SHIFT, 4, 0), |
586 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA1_SHIFT, 4, 0), |
587 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_AES_SHIFT, 4, 0), |
588 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SEVL_SHIFT, 4, 0), |
589 | ARM64_FTR_END, |
590 | }; |
591 | |
592 | static const struct arm64_ftr_bits ftr_id_mmfr4[] = { |
593 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_EVT_SHIFT, 4, 0), |
594 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CCIDX_SHIFT, 4, 0), |
595 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_LSM_SHIFT, 4, 0), |
596 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_HPDS_SHIFT, 4, 0), |
597 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CnP_SHIFT, 4, 0), |
598 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_XNX_SHIFT, 4, 0), |
599 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_AC2_SHIFT, 4, 0), |
600 | |
601 | /* |
602 | * SpecSEI = 1 indicates that the PE might generate an SError on an |
603 | * external abort on speculative read. It is safe to assume that an |
604 | * SError might be generated than it will not be. Hence it has been |
605 | * classified as FTR_HIGHER_SAFE. |
606 | */ |
607 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_EL1_SpecSEI_SHIFT, 4, 0), |
608 | ARM64_FTR_END, |
609 | }; |
610 | |
611 | static const struct arm64_ftr_bits ftr_id_isar4[] = { |
612 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SWP_frac_SHIFT, 4, 0), |
613 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_PSR_M_SHIFT, 4, 0), |
614 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SynchPrim_frac_SHIFT, 4, 0), |
615 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Barrier_SHIFT, 4, 0), |
616 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SMC_SHIFT, 4, 0), |
617 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Writeback_SHIFT, 4, 0), |
618 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_WithShifts_SHIFT, 4, 0), |
619 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Unpriv_SHIFT, 4, 0), |
620 | ARM64_FTR_END, |
621 | }; |
622 | |
623 | static const struct arm64_ftr_bits ftr_id_mmfr5[] = { |
624 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_EL1_ETS_SHIFT, 4, 0), |
625 | ARM64_FTR_END, |
626 | }; |
627 | |
628 | static const struct arm64_ftr_bits ftr_id_isar6[] = { |
629 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_I8MM_SHIFT, 4, 0), |
630 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_BF16_SHIFT, 4, 0), |
631 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SPECRES_SHIFT, 4, 0), |
632 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SB_SHIFT, 4, 0), |
633 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_FHM_SHIFT, 4, 0), |
634 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_DP_SHIFT, 4, 0), |
635 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_JSCVT_SHIFT, 4, 0), |
636 | ARM64_FTR_END, |
637 | }; |
638 | |
639 | static const struct arm64_ftr_bits ftr_id_pfr0[] = { |
640 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_DIT_SHIFT, 4, 0), |
641 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_CSV2_SHIFT, 4, 0), |
642 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State3_SHIFT, 4, 0), |
643 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State2_SHIFT, 4, 0), |
644 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State1_SHIFT, 4, 0), |
645 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State0_SHIFT, 4, 0), |
646 | ARM64_FTR_END, |
647 | }; |
648 | |
649 | static const struct arm64_ftr_bits ftr_id_pfr1[] = { |
650 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GIC_SHIFT, 4, 0), |
651 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virt_frac_SHIFT, 4, 0), |
652 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Sec_frac_SHIFT, 4, 0), |
653 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GenTimer_SHIFT, 4, 0), |
654 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virtualization_SHIFT, 4, 0), |
655 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_MProgMod_SHIFT, 4, 0), |
656 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Security_SHIFT, 4, 0), |
657 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_ProgMod_SHIFT, 4, 0), |
658 | ARM64_FTR_END, |
659 | }; |
660 | |
661 | static const struct arm64_ftr_bits ftr_id_pfr2[] = { |
662 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_SSBS_SHIFT, 4, 0), |
663 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_CSV3_SHIFT, 4, 0), |
664 | ARM64_FTR_END, |
665 | }; |
666 | |
667 | static const struct arm64_ftr_bits ftr_id_dfr0[] = { |
668 | /* [31:28] TraceFilt */ |
669 | S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0), |
670 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MProfDbg_SHIFT, 4, 0), |
671 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapTrc_SHIFT, 4, 0), |
672 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopTrc_SHIFT, 4, 0), |
673 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapDbg_SHIFT, 4, 0), |
674 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopSDbg_SHIFT, 4, 0), |
675 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopDbg_SHIFT, 4, 0), |
676 | ARM64_FTR_END, |
677 | }; |
678 | |
679 | static const struct arm64_ftr_bits ftr_id_dfr1[] = { |
680 | S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_EL1_MTPMU_SHIFT, 4, 0), |
681 | ARM64_FTR_END, |
682 | }; |
683 | |
684 | /* |
685 | * Common ftr bits for a 32bit register with all hidden, strict |
686 | * attributes, with 4bit feature fields and a default safe value of |
687 | * 0. Covers the following 32bit registers: |
688 | * id_isar[1-3], id_mmfr[1-3] |
689 | */ |
690 | static const struct arm64_ftr_bits ftr_generic_32bits[] = { |
691 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0), |
692 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0), |
693 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0), |
694 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0), |
695 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0), |
696 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0), |
697 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), |
698 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), |
699 | ARM64_FTR_END, |
700 | }; |
701 | |
702 | /* Table for a single 32bit feature value */ |
703 | static const struct arm64_ftr_bits ftr_single32[] = { |
704 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 32, 0), |
705 | ARM64_FTR_END, |
706 | }; |
707 | |
708 | static const struct arm64_ftr_bits ftr_raz[] = { |
709 | ARM64_FTR_END, |
710 | }; |
711 | |
712 | #define __ARM64_FTR_REG_OVERRIDE(id_str, id, table, ovr) { \ |
713 | .sys_id = id, \ |
714 | .reg = &(struct arm64_ftr_reg){ \ |
715 | .name = id_str, \ |
716 | .override = (ovr), \ |
717 | .ftr_bits = &((table)[0]), \ |
718 | }} |
719 | |
720 | #define ARM64_FTR_REG_OVERRIDE(id, table, ovr) \ |
721 | __ARM64_FTR_REG_OVERRIDE(#id, id, table, ovr) |
722 | |
723 | #define ARM64_FTR_REG(id, table) \ |
724 | __ARM64_FTR_REG_OVERRIDE(#id, id, table, &no_override) |
725 | |
726 | struct arm64_ftr_override id_aa64mmfr0_override; |
727 | struct arm64_ftr_override id_aa64mmfr1_override; |
728 | struct arm64_ftr_override id_aa64mmfr2_override; |
729 | struct arm64_ftr_override id_aa64pfr0_override; |
730 | struct arm64_ftr_override id_aa64pfr1_override; |
731 | struct arm64_ftr_override id_aa64zfr0_override; |
732 | struct arm64_ftr_override id_aa64smfr0_override; |
733 | struct arm64_ftr_override id_aa64isar1_override; |
734 | struct arm64_ftr_override id_aa64isar2_override; |
735 | |
736 | struct arm64_ftr_override arm64_sw_feature_override; |
737 | |
738 | static const struct __ftr_reg_entry { |
739 | u32 sys_id; |
740 | struct arm64_ftr_reg *reg; |
741 | } arm64_ftr_regs[] = { |
742 | |
743 | /* Op1 = 0, CRn = 0, CRm = 1 */ |
744 | ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0), |
745 | ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_id_pfr1), |
746 | ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_id_dfr0), |
747 | ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0), |
748 | ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits), |
749 | ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits), |
750 | ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits), |
751 | |
752 | /* Op1 = 0, CRn = 0, CRm = 2 */ |
753 | ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_id_isar0), |
754 | ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits), |
755 | ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits), |
756 | ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits), |
757 | ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_id_isar4), |
758 | ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5), |
759 | ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4), |
760 | ARM64_FTR_REG(SYS_ID_ISAR6_EL1, ftr_id_isar6), |
761 | |
762 | /* Op1 = 0, CRn = 0, CRm = 3 */ |
763 | ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_mvfr0), |
764 | ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_mvfr1), |
765 | ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2), |
766 | ARM64_FTR_REG(SYS_ID_PFR2_EL1, ftr_id_pfr2), |
767 | ARM64_FTR_REG(SYS_ID_DFR1_EL1, ftr_id_dfr1), |
768 | ARM64_FTR_REG(SYS_ID_MMFR5_EL1, ftr_id_mmfr5), |
769 | |
770 | /* Op1 = 0, CRn = 0, CRm = 4 */ |
771 | ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0, |
772 | &id_aa64pfr0_override), |
773 | ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1, |
774 | &id_aa64pfr1_override), |
775 | ARM64_FTR_REG(SYS_ID_AA64PFR2_EL1, ftr_id_aa64pfr2), |
776 | ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ZFR0_EL1, ftr_id_aa64zfr0, |
777 | &id_aa64zfr0_override), |
778 | ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64SMFR0_EL1, ftr_id_aa64smfr0, |
779 | &id_aa64smfr0_override), |
780 | ARM64_FTR_REG(SYS_ID_AA64FPFR0_EL1, ftr_id_aa64fpfr0), |
781 | |
782 | /* Op1 = 0, CRn = 0, CRm = 5 */ |
783 | ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0), |
784 | ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_raz), |
785 | |
786 | /* Op1 = 0, CRn = 0, CRm = 6 */ |
787 | ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0), |
788 | ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1, |
789 | &id_aa64isar1_override), |
790 | ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2, |
791 | &id_aa64isar2_override), |
792 | ARM64_FTR_REG(SYS_ID_AA64ISAR3_EL1, ftr_id_aa64isar3), |
793 | |
794 | /* Op1 = 0, CRn = 0, CRm = 7 */ |
795 | ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0, |
796 | &id_aa64mmfr0_override), |
797 | ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1, |
798 | &id_aa64mmfr1_override), |
799 | ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2, |
800 | &id_aa64mmfr2_override), |
801 | ARM64_FTR_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3), |
802 | ARM64_FTR_REG(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4), |
803 | |
804 | /* Op1 = 1, CRn = 0, CRm = 0 */ |
805 | ARM64_FTR_REG(SYS_GMID_EL1, ftr_gmid), |
806 | |
807 | /* Op1 = 3, CRn = 0, CRm = 0 */ |
808 | { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 }, |
809 | ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid), |
810 | |
811 | /* Op1 = 3, CRn = 14, CRm = 0 */ |
812 | ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_single32), |
813 | }; |
814 | |
815 | static int search_cmp_ftr_reg(const void *id, const void *regp) |
816 | { |
817 | return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id; |
818 | } |
819 | |
820 | /* |
821 | * get_arm64_ftr_reg_nowarn - Looks up a feature register entry using |
822 | * its sys_reg() encoding. With the array arm64_ftr_regs sorted in the |
823 | * ascending order of sys_id, we use binary search to find a matching |
824 | * entry. |
825 | * |
826 | * returns - Upon success, matching ftr_reg entry for id. |
827 | * - NULL on failure. It is upto the caller to decide |
828 | * the impact of a failure. |
829 | */ |
830 | static struct arm64_ftr_reg *get_arm64_ftr_reg_nowarn(u32 sys_id) |
831 | { |
832 | const struct __ftr_reg_entry *ret; |
833 | |
834 | ret = bsearch((const void *)(unsigned long)sys_id, |
835 | arm64_ftr_regs, |
836 | ARRAY_SIZE(arm64_ftr_regs), |
837 | sizeof(arm64_ftr_regs[0]), |
838 | search_cmp_ftr_reg); |
839 | if (ret) |
840 | return ret->reg; |
841 | return NULL; |
842 | } |
843 | |
844 | /* |
845 | * get_arm64_ftr_reg - Looks up a feature register entry using |
846 | * its sys_reg() encoding. This calls get_arm64_ftr_reg_nowarn(). |
847 | * |
848 | * returns - Upon success, matching ftr_reg entry for id. |
849 | * - NULL on failure but with an WARN_ON(). |
850 | */ |
851 | struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id) |
852 | { |
853 | struct arm64_ftr_reg *reg; |
854 | |
855 | reg = get_arm64_ftr_reg_nowarn(sys_id); |
856 | |
857 | /* |
858 | * Requesting a non-existent register search is an error. Warn |
859 | * and let the caller handle it. |
860 | */ |
861 | WARN_ON(!reg); |
862 | return reg; |
863 | } |
864 | |
865 | static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg, |
866 | s64 ftr_val) |
867 | { |
868 | u64 mask = arm64_ftr_mask(ftrp); |
869 | |
870 | reg &= ~mask; |
871 | reg |= (ftr_val << ftrp->shift) & mask; |
872 | return reg; |
873 | } |
874 | |
875 | s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, |
876 | s64 cur) |
877 | { |
878 | s64 ret = 0; |
879 | |
880 | switch (ftrp->type) { |
881 | case FTR_EXACT: |
882 | ret = ftrp->safe_val; |
883 | break; |
884 | case FTR_LOWER_SAFE: |
885 | ret = min(new, cur); |
886 | break; |
887 | case FTR_HIGHER_OR_ZERO_SAFE: |
888 | if (!cur || !new) |
889 | break; |
890 | fallthrough; |
891 | case FTR_HIGHER_SAFE: |
892 | ret = max(new, cur); |
893 | break; |
894 | default: |
895 | BUG(); |
896 | } |
897 | |
898 | return ret; |
899 | } |
900 | |
901 | static void __init sort_ftr_regs(void) |
902 | { |
903 | unsigned int i; |
904 | |
905 | for (i = 0; i < ARRAY_SIZE(arm64_ftr_regs); i++) { |
906 | const struct arm64_ftr_reg *ftr_reg = arm64_ftr_regs[i].reg; |
907 | const struct arm64_ftr_bits *ftr_bits = ftr_reg->ftr_bits; |
908 | unsigned int j = 0; |
909 | |
910 | /* |
911 | * Features here must be sorted in descending order with respect |
912 | * to their shift values and should not overlap with each other. |
913 | */ |
914 | for (; ftr_bits->width != 0; ftr_bits++, j++) { |
915 | unsigned int width = ftr_reg->ftr_bits[j].width; |
916 | unsigned int shift = ftr_reg->ftr_bits[j].shift; |
917 | unsigned int prev_shift; |
918 | |
919 | WARN((shift + width) > 64, |
920 | "%s has invalid feature at shift %d\n" , |
921 | ftr_reg->name, shift); |
922 | |
923 | /* |
924 | * Skip the first feature. There is nothing to |
925 | * compare against for now. |
926 | */ |
927 | if (j == 0) |
928 | continue; |
929 | |
930 | prev_shift = ftr_reg->ftr_bits[j - 1].shift; |
931 | WARN((shift + width) > prev_shift, |
932 | "%s has feature overlap at shift %d\n" , |
933 | ftr_reg->name, shift); |
934 | } |
935 | |
936 | /* |
937 | * Skip the first register. There is nothing to |
938 | * compare against for now. |
939 | */ |
940 | if (i == 0) |
941 | continue; |
942 | /* |
943 | * Registers here must be sorted in ascending order with respect |
944 | * to sys_id for subsequent binary search in get_arm64_ftr_reg() |
945 | * to work correctly. |
946 | */ |
947 | BUG_ON(arm64_ftr_regs[i].sys_id <= arm64_ftr_regs[i - 1].sys_id); |
948 | } |
949 | } |
950 | |
951 | /* |
952 | * Initialise the CPU feature register from Boot CPU values. |
953 | * Also initiliases the strict_mask for the register. |
954 | * Any bits that are not covered by an arm64_ftr_bits entry are considered |
955 | * RES0 for the system-wide value, and must strictly match. |
956 | */ |
957 | static void init_cpu_ftr_reg(u32 sys_reg, u64 new) |
958 | { |
959 | u64 val = 0; |
960 | u64 strict_mask = ~0x0ULL; |
961 | u64 user_mask = 0; |
962 | u64 valid_mask = 0; |
963 | |
964 | const struct arm64_ftr_bits *ftrp; |
965 | struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_id: sys_reg); |
966 | |
967 | if (!reg) |
968 | return; |
969 | |
970 | for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) { |
971 | u64 ftr_mask = arm64_ftr_mask(ftrp); |
972 | s64 ftr_new = arm64_ftr_value(ftrp, new); |
973 | s64 ftr_ovr = arm64_ftr_value(ftrp, reg->override->val); |
974 | |
975 | if ((ftr_mask & reg->override->mask) == ftr_mask) { |
976 | s64 tmp = arm64_ftr_safe_value(ftrp, new: ftr_ovr, cur: ftr_new); |
977 | char *str = NULL; |
978 | |
979 | if (ftr_ovr != tmp) { |
980 | /* Unsafe, remove the override */ |
981 | reg->override->mask &= ~ftr_mask; |
982 | reg->override->val &= ~ftr_mask; |
983 | tmp = ftr_ovr; |
984 | str = "ignoring override" ; |
985 | } else if (ftr_new != tmp) { |
986 | /* Override was valid */ |
987 | ftr_new = tmp; |
988 | str = "forced" ; |
989 | } else if (ftr_ovr == tmp) { |
990 | /* Override was the safe value */ |
991 | str = "already set" ; |
992 | } |
993 | |
994 | if (str) |
995 | pr_warn("%s[%d:%d]: %s to %llx\n" , |
996 | reg->name, |
997 | ftrp->shift + ftrp->width - 1, |
998 | ftrp->shift, str, |
999 | tmp & (BIT(ftrp->width) - 1)); |
1000 | } else if ((ftr_mask & reg->override->val) == ftr_mask) { |
1001 | reg->override->val &= ~ftr_mask; |
1002 | pr_warn("%s[%d:%d]: impossible override, ignored\n" , |
1003 | reg->name, |
1004 | ftrp->shift + ftrp->width - 1, |
1005 | ftrp->shift); |
1006 | } |
1007 | |
1008 | val = arm64_ftr_set_value(ftrp, reg: val, ftr_val: ftr_new); |
1009 | |
1010 | valid_mask |= ftr_mask; |
1011 | if (!ftrp->strict) |
1012 | strict_mask &= ~ftr_mask; |
1013 | if (ftrp->visible) |
1014 | user_mask |= ftr_mask; |
1015 | else |
1016 | reg->user_val = arm64_ftr_set_value(ftrp, |
1017 | reg: reg->user_val, |
1018 | ftr_val: ftrp->safe_val); |
1019 | } |
1020 | |
1021 | val &= valid_mask; |
1022 | |
1023 | reg->sys_val = val; |
1024 | reg->strict_mask = strict_mask; |
1025 | reg->user_mask = user_mask; |
1026 | } |
1027 | |
1028 | extern const struct arm64_cpu_capabilities arm64_errata[]; |
1029 | static const struct arm64_cpu_capabilities arm64_features[]; |
1030 | |
1031 | static void __init |
1032 | init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps) |
1033 | { |
1034 | for (; caps->matches; caps++) { |
1035 | if (WARN(caps->capability >= ARM64_NCAPS, |
1036 | "Invalid capability %d\n" , caps->capability)) |
1037 | continue; |
1038 | if (WARN(cpucap_ptrs[caps->capability], |
1039 | "Duplicate entry for capability %d\n" , |
1040 | caps->capability)) |
1041 | continue; |
1042 | cpucap_ptrs[caps->capability] = caps; |
1043 | } |
1044 | } |
1045 | |
1046 | static void __init init_cpucap_indirect_list(void) |
1047 | { |
1048 | init_cpucap_indirect_list_from_array(caps: arm64_features); |
1049 | init_cpucap_indirect_list_from_array(caps: arm64_errata); |
1050 | } |
1051 | |
1052 | static void __init setup_boot_cpu_capabilities(void); |
1053 | |
1054 | static void init_32bit_cpu_features(struct cpuinfo_32bit *info) |
1055 | { |
1056 | init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0); |
1057 | init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1); |
1058 | init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0); |
1059 | init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1); |
1060 | init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2); |
1061 | init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3); |
1062 | init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4); |
1063 | init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5); |
1064 | init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6); |
1065 | init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0); |
1066 | init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1); |
1067 | init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2); |
1068 | init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3); |
1069 | init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4); |
1070 | init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5); |
1071 | init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0); |
1072 | init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1); |
1073 | init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2); |
1074 | init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0); |
1075 | init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1); |
1076 | init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2); |
1077 | } |
1078 | |
1079 | #ifdef CONFIG_ARM64_PSEUDO_NMI |
1080 | static bool enable_pseudo_nmi; |
1081 | |
1082 | static int __init early_enable_pseudo_nmi(char *p) |
1083 | { |
1084 | return kstrtobool(p, &enable_pseudo_nmi); |
1085 | } |
1086 | early_param("irqchip.gicv3_pseudo_nmi" , early_enable_pseudo_nmi); |
1087 | |
1088 | static __init void detect_system_supports_pseudo_nmi(void) |
1089 | { |
1090 | struct device_node *np; |
1091 | |
1092 | if (!enable_pseudo_nmi) |
1093 | return; |
1094 | |
1095 | /* |
1096 | * Detect broken MediaTek firmware that doesn't properly save and |
1097 | * restore GIC priorities. |
1098 | */ |
1099 | np = of_find_compatible_node(NULL, NULL, "arm,gic-v3" ); |
1100 | if (np && of_property_read_bool(np, "mediatek,broken-save-restore-fw" )) { |
1101 | pr_info("Pseudo-NMI disabled due to MediaTek Chromebook GICR save problem\n" ); |
1102 | enable_pseudo_nmi = false; |
1103 | } |
1104 | of_node_put(np); |
1105 | } |
1106 | #else /* CONFIG_ARM64_PSEUDO_NMI */ |
1107 | static inline void detect_system_supports_pseudo_nmi(void) { } |
1108 | #endif |
1109 | |
1110 | void __init init_cpu_features(struct cpuinfo_arm64 *info) |
1111 | { |
1112 | /* Before we start using the tables, make sure it is sorted */ |
1113 | sort_ftr_regs(); |
1114 | |
1115 | init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr); |
1116 | init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid); |
1117 | init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq); |
1118 | init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0); |
1119 | init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1); |
1120 | init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0); |
1121 | init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1); |
1122 | init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2); |
1123 | init_cpu_ftr_reg(SYS_ID_AA64ISAR3_EL1, info->reg_id_aa64isar3); |
1124 | init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0); |
1125 | init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1); |
1126 | init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2); |
1127 | init_cpu_ftr_reg(SYS_ID_AA64MMFR3_EL1, info->reg_id_aa64mmfr3); |
1128 | init_cpu_ftr_reg(SYS_ID_AA64MMFR4_EL1, info->reg_id_aa64mmfr4); |
1129 | init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0); |
1130 | init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1); |
1131 | init_cpu_ftr_reg(SYS_ID_AA64PFR2_EL1, info->reg_id_aa64pfr2); |
1132 | init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0); |
1133 | init_cpu_ftr_reg(SYS_ID_AA64SMFR0_EL1, info->reg_id_aa64smfr0); |
1134 | init_cpu_ftr_reg(SYS_ID_AA64FPFR0_EL1, info->reg_id_aa64fpfr0); |
1135 | |
1136 | if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) |
1137 | init_32bit_cpu_features(info: &info->aarch32); |
1138 | |
1139 | if (IS_ENABLED(CONFIG_ARM64_SVE) && |
1140 | id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { |
1141 | unsigned long cpacr = cpacr_save_enable_kernel_sve(); |
1142 | |
1143 | vec_init_vq_map(ARM64_VEC_SVE); |
1144 | |
1145 | cpacr_restore(cpacr); |
1146 | } |
1147 | |
1148 | if (IS_ENABLED(CONFIG_ARM64_SME) && |
1149 | id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) { |
1150 | unsigned long cpacr = cpacr_save_enable_kernel_sme(); |
1151 | |
1152 | /* |
1153 | * We mask out SMPS since even if the hardware |
1154 | * supports priorities the kernel does not at present |
1155 | * and we block access to them. |
1156 | */ |
1157 | info->reg_smidr = read_cpuid(SMIDR_EL1) & ~SMIDR_EL1_SMPS; |
1158 | vec_init_vq_map(ARM64_VEC_SME); |
1159 | |
1160 | cpacr_restore(cpacr); |
1161 | } |
1162 | |
1163 | if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) |
1164 | init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid); |
1165 | } |
1166 | |
1167 | static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new) |
1168 | { |
1169 | const struct arm64_ftr_bits *ftrp; |
1170 | |
1171 | for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) { |
1172 | s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val); |
1173 | s64 ftr_new = arm64_ftr_value(ftrp, new); |
1174 | |
1175 | if (ftr_cur == ftr_new) |
1176 | continue; |
1177 | /* Find a safe value */ |
1178 | ftr_new = arm64_ftr_safe_value(ftrp, new: ftr_new, cur: ftr_cur); |
1179 | reg->sys_val = arm64_ftr_set_value(ftrp, reg: reg->sys_val, ftr_val: ftr_new); |
1180 | } |
1181 | |
1182 | } |
1183 | |
1184 | static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot) |
1185 | { |
1186 | struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id); |
1187 | |
1188 | if (!regp) |
1189 | return 0; |
1190 | |
1191 | update_cpu_ftr_reg(reg: regp, new: val); |
1192 | if ((boot & regp->strict_mask) == (val & regp->strict_mask)) |
1193 | return 0; |
1194 | pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n" , |
1195 | regp->name, boot, cpu, val); |
1196 | return 1; |
1197 | } |
1198 | |
1199 | static void relax_cpu_ftr_reg(u32 sys_id, int field) |
1200 | { |
1201 | const struct arm64_ftr_bits *ftrp; |
1202 | struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id); |
1203 | |
1204 | if (!regp) |
1205 | return; |
1206 | |
1207 | for (ftrp = regp->ftr_bits; ftrp->width; ftrp++) { |
1208 | if (ftrp->shift == field) { |
1209 | regp->strict_mask &= ~arm64_ftr_mask(ftrp); |
1210 | break; |
1211 | } |
1212 | } |
1213 | |
1214 | /* Bogus field? */ |
1215 | WARN_ON(!ftrp->width); |
1216 | } |
1217 | |
1218 | static void lazy_init_32bit_cpu_features(struct cpuinfo_arm64 *info, |
1219 | struct cpuinfo_arm64 *boot) |
1220 | { |
1221 | static bool boot_cpu_32bit_regs_overridden = false; |
1222 | |
1223 | if (!allow_mismatched_32bit_el0 || boot_cpu_32bit_regs_overridden) |
1224 | return; |
1225 | |
1226 | if (id_aa64pfr0_32bit_el0(boot->reg_id_aa64pfr0)) |
1227 | return; |
1228 | |
1229 | boot->aarch32 = info->aarch32; |
1230 | init_32bit_cpu_features(info: &boot->aarch32); |
1231 | boot_cpu_32bit_regs_overridden = true; |
1232 | } |
1233 | |
1234 | static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info, |
1235 | struct cpuinfo_32bit *boot) |
1236 | { |
1237 | int taint = 0; |
1238 | u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); |
1239 | |
1240 | /* |
1241 | * If we don't have AArch32 at EL1, then relax the strictness of |
1242 | * EL1-dependent register fields to avoid spurious sanity check fails. |
1243 | */ |
1244 | if (!id_aa64pfr0_32bit_el1(pfr0)) { |
1245 | relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_EL1_SMC_SHIFT); |
1246 | relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virt_frac_SHIFT); |
1247 | relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Sec_frac_SHIFT); |
1248 | relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virtualization_SHIFT); |
1249 | relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Security_SHIFT); |
1250 | relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_ProgMod_SHIFT); |
1251 | } |
1252 | |
1253 | taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu, |
1254 | info->reg_id_dfr0, boot->reg_id_dfr0); |
1255 | taint |= check_update_ftr_reg(SYS_ID_DFR1_EL1, cpu, |
1256 | info->reg_id_dfr1, boot->reg_id_dfr1); |
1257 | taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu, |
1258 | info->reg_id_isar0, boot->reg_id_isar0); |
1259 | taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu, |
1260 | info->reg_id_isar1, boot->reg_id_isar1); |
1261 | taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu, |
1262 | info->reg_id_isar2, boot->reg_id_isar2); |
1263 | taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu, |
1264 | info->reg_id_isar3, boot->reg_id_isar3); |
1265 | taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu, |
1266 | info->reg_id_isar4, boot->reg_id_isar4); |
1267 | taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu, |
1268 | info->reg_id_isar5, boot->reg_id_isar5); |
1269 | taint |= check_update_ftr_reg(SYS_ID_ISAR6_EL1, cpu, |
1270 | info->reg_id_isar6, boot->reg_id_isar6); |
1271 | |
1272 | /* |
1273 | * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and |
1274 | * ACTLR formats could differ across CPUs and therefore would have to |
1275 | * be trapped for virtualization anyway. |
1276 | */ |
1277 | taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu, |
1278 | info->reg_id_mmfr0, boot->reg_id_mmfr0); |
1279 | taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu, |
1280 | info->reg_id_mmfr1, boot->reg_id_mmfr1); |
1281 | taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu, |
1282 | info->reg_id_mmfr2, boot->reg_id_mmfr2); |
1283 | taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu, |
1284 | info->reg_id_mmfr3, boot->reg_id_mmfr3); |
1285 | taint |= check_update_ftr_reg(SYS_ID_MMFR4_EL1, cpu, |
1286 | info->reg_id_mmfr4, boot->reg_id_mmfr4); |
1287 | taint |= check_update_ftr_reg(SYS_ID_MMFR5_EL1, cpu, |
1288 | info->reg_id_mmfr5, boot->reg_id_mmfr5); |
1289 | taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu, |
1290 | info->reg_id_pfr0, boot->reg_id_pfr0); |
1291 | taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu, |
1292 | info->reg_id_pfr1, boot->reg_id_pfr1); |
1293 | taint |= check_update_ftr_reg(SYS_ID_PFR2_EL1, cpu, |
1294 | info->reg_id_pfr2, boot->reg_id_pfr2); |
1295 | taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu, |
1296 | info->reg_mvfr0, boot->reg_mvfr0); |
1297 | taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu, |
1298 | info->reg_mvfr1, boot->reg_mvfr1); |
1299 | taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu, |
1300 | info->reg_mvfr2, boot->reg_mvfr2); |
1301 | |
1302 | return taint; |
1303 | } |
1304 | |
1305 | /* |
1306 | * Update system wide CPU feature registers with the values from a |
1307 | * non-boot CPU. Also performs SANITY checks to make sure that there |
1308 | * aren't any insane variations from that of the boot CPU. |
1309 | */ |
1310 | void update_cpu_features(int cpu, |
1311 | struct cpuinfo_arm64 *info, |
1312 | struct cpuinfo_arm64 *boot) |
1313 | { |
1314 | int taint = 0; |
1315 | |
1316 | /* |
1317 | * The kernel can handle differing I-cache policies, but otherwise |
1318 | * caches should look identical. Userspace JITs will make use of |
1319 | * *minLine. |
1320 | */ |
1321 | taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu, |
1322 | info->reg_ctr, boot->reg_ctr); |
1323 | |
1324 | /* |
1325 | * Userspace may perform DC ZVA instructions. Mismatched block sizes |
1326 | * could result in too much or too little memory being zeroed if a |
1327 | * process is preempted and migrated between CPUs. |
1328 | */ |
1329 | taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu, |
1330 | info->reg_dczid, boot->reg_dczid); |
1331 | |
1332 | /* If different, timekeeping will be broken (especially with KVM) */ |
1333 | taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu, |
1334 | info->reg_cntfrq, boot->reg_cntfrq); |
1335 | |
1336 | /* |
1337 | * The kernel uses self-hosted debug features and expects CPUs to |
1338 | * support identical debug features. We presently need CTX_CMPs, WRPs, |
1339 | * and BRPs to be identical. |
1340 | * ID_AA64DFR1 is currently RES0. |
1341 | */ |
1342 | taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu, |
1343 | info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0); |
1344 | taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu, |
1345 | info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1); |
1346 | /* |
1347 | * Even in big.LITTLE, processors should be identical instruction-set |
1348 | * wise. |
1349 | */ |
1350 | taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu, |
1351 | info->reg_id_aa64isar0, boot->reg_id_aa64isar0); |
1352 | taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu, |
1353 | info->reg_id_aa64isar1, boot->reg_id_aa64isar1); |
1354 | taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu, |
1355 | info->reg_id_aa64isar2, boot->reg_id_aa64isar2); |
1356 | taint |= check_update_ftr_reg(SYS_ID_AA64ISAR3_EL1, cpu, |
1357 | info->reg_id_aa64isar3, boot->reg_id_aa64isar3); |
1358 | |
1359 | /* |
1360 | * Differing PARange support is fine as long as all peripherals and |
1361 | * memory are mapped within the minimum PARange of all CPUs. |
1362 | * Linux should not care about secure memory. |
1363 | */ |
1364 | taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu, |
1365 | info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0); |
1366 | taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu, |
1367 | info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1); |
1368 | taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu, |
1369 | info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2); |
1370 | taint |= check_update_ftr_reg(SYS_ID_AA64MMFR3_EL1, cpu, |
1371 | info->reg_id_aa64mmfr3, boot->reg_id_aa64mmfr3); |
1372 | |
1373 | taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu, |
1374 | info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0); |
1375 | taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu, |
1376 | info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1); |
1377 | taint |= check_update_ftr_reg(SYS_ID_AA64PFR2_EL1, cpu, |
1378 | info->reg_id_aa64pfr2, boot->reg_id_aa64pfr2); |
1379 | |
1380 | taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu, |
1381 | info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0); |
1382 | |
1383 | taint |= check_update_ftr_reg(SYS_ID_AA64SMFR0_EL1, cpu, |
1384 | info->reg_id_aa64smfr0, boot->reg_id_aa64smfr0); |
1385 | |
1386 | taint |= check_update_ftr_reg(SYS_ID_AA64FPFR0_EL1, cpu, |
1387 | info->reg_id_aa64fpfr0, boot->reg_id_aa64fpfr0); |
1388 | |
1389 | /* Probe vector lengths */ |
1390 | if (IS_ENABLED(CONFIG_ARM64_SVE) && |
1391 | id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) { |
1392 | if (!system_capabilities_finalized()) { |
1393 | unsigned long cpacr = cpacr_save_enable_kernel_sve(); |
1394 | |
1395 | vec_update_vq_map(ARM64_VEC_SVE); |
1396 | |
1397 | cpacr_restore(cpacr); |
1398 | } |
1399 | } |
1400 | |
1401 | if (IS_ENABLED(CONFIG_ARM64_SME) && |
1402 | id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) { |
1403 | unsigned long cpacr = cpacr_save_enable_kernel_sme(); |
1404 | |
1405 | /* |
1406 | * We mask out SMPS since even if the hardware |
1407 | * supports priorities the kernel does not at present |
1408 | * and we block access to them. |
1409 | */ |
1410 | info->reg_smidr = read_cpuid(SMIDR_EL1) & ~SMIDR_EL1_SMPS; |
1411 | |
1412 | /* Probe vector lengths */ |
1413 | if (!system_capabilities_finalized()) |
1414 | vec_update_vq_map(ARM64_VEC_SME); |
1415 | |
1416 | cpacr_restore(cpacr); |
1417 | } |
1418 | |
1419 | /* |
1420 | * The kernel uses the LDGM/STGM instructions and the number of tags |
1421 | * they read/write depends on the GMID_EL1.BS field. Check that the |
1422 | * value is the same on all CPUs. |
1423 | */ |
1424 | if (IS_ENABLED(CONFIG_ARM64_MTE) && |
1425 | id_aa64pfr1_mte(info->reg_id_aa64pfr1)) { |
1426 | taint |= check_update_ftr_reg(SYS_GMID_EL1, cpu, |
1427 | info->reg_gmid, boot->reg_gmid); |
1428 | } |
1429 | |
1430 | /* |
1431 | * If we don't have AArch32 at all then skip the checks entirely |
1432 | * as the register values may be UNKNOWN and we're not going to be |
1433 | * using them for anything. |
1434 | * |
1435 | * This relies on a sanitised view of the AArch64 ID registers |
1436 | * (e.g. SYS_ID_AA64PFR0_EL1), so we call it last. |
1437 | */ |
1438 | if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { |
1439 | lazy_init_32bit_cpu_features(info, boot); |
1440 | taint |= update_32bit_cpu_features(cpu, info: &info->aarch32, |
1441 | boot: &boot->aarch32); |
1442 | } |
1443 | |
1444 | /* |
1445 | * Mismatched CPU features are a recipe for disaster. Don't even |
1446 | * pretend to support them. |
1447 | */ |
1448 | if (taint) { |
1449 | pr_warn_once("Unsupported CPU feature variation detected.\n" ); |
1450 | add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK); |
1451 | } |
1452 | } |
1453 | |
1454 | u64 read_sanitised_ftr_reg(u32 id) |
1455 | { |
1456 | struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id: id); |
1457 | |
1458 | if (!regp) |
1459 | return 0; |
1460 | return regp->sys_val; |
1461 | } |
1462 | EXPORT_SYMBOL_GPL(read_sanitised_ftr_reg); |
1463 | |
1464 | #define read_sysreg_case(r) \ |
1465 | case r: val = read_sysreg_s(r); break; |
1466 | |
1467 | /* |
1468 | * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated. |
1469 | * Read the system register on the current CPU |
1470 | */ |
1471 | u64 __read_sysreg_by_encoding(u32 sys_id) |
1472 | { |
1473 | struct arm64_ftr_reg *regp; |
1474 | u64 val; |
1475 | |
1476 | switch (sys_id) { |
1477 | read_sysreg_case(SYS_ID_PFR0_EL1); |
1478 | read_sysreg_case(SYS_ID_PFR1_EL1); |
1479 | read_sysreg_case(SYS_ID_PFR2_EL1); |
1480 | read_sysreg_case(SYS_ID_DFR0_EL1); |
1481 | read_sysreg_case(SYS_ID_DFR1_EL1); |
1482 | read_sysreg_case(SYS_ID_MMFR0_EL1); |
1483 | read_sysreg_case(SYS_ID_MMFR1_EL1); |
1484 | read_sysreg_case(SYS_ID_MMFR2_EL1); |
1485 | read_sysreg_case(SYS_ID_MMFR3_EL1); |
1486 | read_sysreg_case(SYS_ID_MMFR4_EL1); |
1487 | read_sysreg_case(SYS_ID_MMFR5_EL1); |
1488 | read_sysreg_case(SYS_ID_ISAR0_EL1); |
1489 | read_sysreg_case(SYS_ID_ISAR1_EL1); |
1490 | read_sysreg_case(SYS_ID_ISAR2_EL1); |
1491 | read_sysreg_case(SYS_ID_ISAR3_EL1); |
1492 | read_sysreg_case(SYS_ID_ISAR4_EL1); |
1493 | read_sysreg_case(SYS_ID_ISAR5_EL1); |
1494 | read_sysreg_case(SYS_ID_ISAR6_EL1); |
1495 | read_sysreg_case(SYS_MVFR0_EL1); |
1496 | read_sysreg_case(SYS_MVFR1_EL1); |
1497 | read_sysreg_case(SYS_MVFR2_EL1); |
1498 | |
1499 | read_sysreg_case(SYS_ID_AA64PFR0_EL1); |
1500 | read_sysreg_case(SYS_ID_AA64PFR1_EL1); |
1501 | read_sysreg_case(SYS_ID_AA64PFR2_EL1); |
1502 | read_sysreg_case(SYS_ID_AA64ZFR0_EL1); |
1503 | read_sysreg_case(SYS_ID_AA64SMFR0_EL1); |
1504 | read_sysreg_case(SYS_ID_AA64FPFR0_EL1); |
1505 | read_sysreg_case(SYS_ID_AA64DFR0_EL1); |
1506 | read_sysreg_case(SYS_ID_AA64DFR1_EL1); |
1507 | read_sysreg_case(SYS_ID_AA64MMFR0_EL1); |
1508 | read_sysreg_case(SYS_ID_AA64MMFR1_EL1); |
1509 | read_sysreg_case(SYS_ID_AA64MMFR2_EL1); |
1510 | read_sysreg_case(SYS_ID_AA64MMFR3_EL1); |
1511 | read_sysreg_case(SYS_ID_AA64MMFR4_EL1); |
1512 | read_sysreg_case(SYS_ID_AA64ISAR0_EL1); |
1513 | read_sysreg_case(SYS_ID_AA64ISAR1_EL1); |
1514 | read_sysreg_case(SYS_ID_AA64ISAR2_EL1); |
1515 | read_sysreg_case(SYS_ID_AA64ISAR3_EL1); |
1516 | |
1517 | read_sysreg_case(SYS_CNTFRQ_EL0); |
1518 | read_sysreg_case(SYS_CTR_EL0); |
1519 | read_sysreg_case(SYS_DCZID_EL0); |
1520 | |
1521 | default: |
1522 | BUG(); |
1523 | return 0; |
1524 | } |
1525 | |
1526 | regp = get_arm64_ftr_reg(sys_id); |
1527 | if (regp) { |
1528 | val &= ~regp->override->mask; |
1529 | val |= (regp->override->val & regp->override->mask); |
1530 | } |
1531 | |
1532 | return val; |
1533 | } |
1534 | |
1535 | #include <linux/irqchip/arm-gic-v3.h> |
1536 | |
1537 | static bool |
1538 | has_always(const struct arm64_cpu_capabilities *entry, int scope) |
1539 | { |
1540 | return true; |
1541 | } |
1542 | |
1543 | static bool |
1544 | feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) |
1545 | { |
1546 | int val, min, max; |
1547 | u64 tmp; |
1548 | |
1549 | val = cpuid_feature_extract_field_width(reg, entry->field_pos, |
1550 | entry->field_width, |
1551 | entry->sign); |
1552 | |
1553 | tmp = entry->min_field_value; |
1554 | tmp <<= entry->field_pos; |
1555 | |
1556 | min = cpuid_feature_extract_field_width(tmp, entry->field_pos, |
1557 | entry->field_width, |
1558 | entry->sign); |
1559 | |
1560 | tmp = entry->max_field_value; |
1561 | tmp <<= entry->field_pos; |
1562 | |
1563 | max = cpuid_feature_extract_field_width(tmp, entry->field_pos, |
1564 | entry->field_width, |
1565 | entry->sign); |
1566 | |
1567 | return val >= min && val <= max; |
1568 | } |
1569 | |
1570 | static u64 |
1571 | read_scoped_sysreg(const struct arm64_cpu_capabilities *entry, int scope) |
1572 | { |
1573 | WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible()); |
1574 | if (scope == SCOPE_SYSTEM) |
1575 | return read_sanitised_ftr_reg(entry->sys_reg); |
1576 | else |
1577 | return __read_sysreg_by_encoding(sys_id: entry->sys_reg); |
1578 | } |
1579 | |
1580 | static bool |
1581 | has_user_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) |
1582 | { |
1583 | int mask; |
1584 | struct arm64_ftr_reg *regp; |
1585 | u64 val = read_scoped_sysreg(entry, scope); |
1586 | |
1587 | regp = get_arm64_ftr_reg(sys_id: entry->sys_reg); |
1588 | if (!regp) |
1589 | return false; |
1590 | |
1591 | mask = cpuid_feature_extract_unsigned_field_width(regp->user_mask, |
1592 | entry->field_pos, |
1593 | entry->field_width); |
1594 | if (!mask) |
1595 | return false; |
1596 | |
1597 | return feature_matches(reg: val, entry); |
1598 | } |
1599 | |
1600 | static bool |
1601 | has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) |
1602 | { |
1603 | u64 val = read_scoped_sysreg(entry, scope); |
1604 | return feature_matches(reg: val, entry); |
1605 | } |
1606 | |
1607 | const struct cpumask *system_32bit_el0_cpumask(void) |
1608 | { |
1609 | if (!system_supports_32bit_el0()) |
1610 | return cpu_none_mask; |
1611 | |
1612 | if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) |
1613 | return cpu_32bit_el0_mask; |
1614 | |
1615 | return cpu_possible_mask; |
1616 | } |
1617 | |
1618 | static int __init parse_32bit_el0_param(char *str) |
1619 | { |
1620 | allow_mismatched_32bit_el0 = true; |
1621 | return 0; |
1622 | } |
1623 | early_param("allow_mismatched_32bit_el0" , parse_32bit_el0_param); |
1624 | |
1625 | static ssize_t aarch32_el0_show(struct device *dev, |
1626 | struct device_attribute *attr, char *buf) |
1627 | { |
1628 | const struct cpumask *mask = system_32bit_el0_cpumask(); |
1629 | |
1630 | return sysfs_emit(buf, fmt: "%*pbl\n" , cpumask_pr_args(mask)); |
1631 | } |
1632 | static const DEVICE_ATTR_RO(aarch32_el0); |
1633 | |
1634 | static int __init aarch32_el0_sysfs_init(void) |
1635 | { |
1636 | struct device *dev_root; |
1637 | int ret = 0; |
1638 | |
1639 | if (!allow_mismatched_32bit_el0) |
1640 | return 0; |
1641 | |
1642 | dev_root = bus_get_dev_root(bus: &cpu_subsys); |
1643 | if (dev_root) { |
1644 | ret = device_create_file(device: dev_root, entry: &dev_attr_aarch32_el0); |
1645 | put_device(dev: dev_root); |
1646 | } |
1647 | return ret; |
1648 | } |
1649 | device_initcall(aarch32_el0_sysfs_init); |
1650 | |
1651 | static bool has_32bit_el0(const struct arm64_cpu_capabilities *entry, int scope) |
1652 | { |
1653 | if (!has_cpuid_feature(entry, scope)) |
1654 | return allow_mismatched_32bit_el0; |
1655 | |
1656 | if (scope == SCOPE_SYSTEM) |
1657 | pr_info("detected: 32-bit EL0 Support\n" ); |
1658 | |
1659 | return true; |
1660 | } |
1661 | |
1662 | static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope) |
1663 | { |
1664 | bool has_sre; |
1665 | |
1666 | if (!has_cpuid_feature(entry, scope)) |
1667 | return false; |
1668 | |
1669 | has_sre = gic_enable_sre(); |
1670 | if (!has_sre) |
1671 | pr_warn_once("%s present but disabled by higher exception level\n" , |
1672 | entry->desc); |
1673 | |
1674 | return has_sre; |
1675 | } |
1676 | |
1677 | static bool has_cache_idc(const struct arm64_cpu_capabilities *entry, |
1678 | int scope) |
1679 | { |
1680 | u64 ctr; |
1681 | |
1682 | if (scope == SCOPE_SYSTEM) |
1683 | ctr = arm64_ftr_reg_ctrel0.sys_val; |
1684 | else |
1685 | ctr = read_cpuid_effective_cachetype(); |
1686 | |
1687 | return ctr & BIT(CTR_EL0_IDC_SHIFT); |
1688 | } |
1689 | |
1690 | static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unused) |
1691 | { |
1692 | /* |
1693 | * If the CPU exposes raw CTR_EL0.IDC = 0, while effectively |
1694 | * CTR_EL0.IDC = 1 (from CLIDR values), we need to trap accesses |
1695 | * to the CTR_EL0 on this CPU and emulate it with the real/safe |
1696 | * value. |
1697 | */ |
1698 | if (!(read_cpuid_cachetype() & BIT(CTR_EL0_IDC_SHIFT))) |
1699 | sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); |
1700 | } |
1701 | |
1702 | static bool has_cache_dic(const struct arm64_cpu_capabilities *entry, |
1703 | int scope) |
1704 | { |
1705 | u64 ctr; |
1706 | |
1707 | if (scope == SCOPE_SYSTEM) |
1708 | ctr = arm64_ftr_reg_ctrel0.sys_val; |
1709 | else |
1710 | ctr = read_cpuid_cachetype(); |
1711 | |
1712 | return ctr & BIT(CTR_EL0_DIC_SHIFT); |
1713 | } |
1714 | |
1715 | static bool __maybe_unused |
1716 | has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope) |
1717 | { |
1718 | /* |
1719 | * Kdump isn't guaranteed to power-off all secondary CPUs, CNP |
1720 | * may share TLB entries with a CPU stuck in the crashed |
1721 | * kernel. |
1722 | */ |
1723 | if (is_kdump_kernel()) |
1724 | return false; |
1725 | |
1726 | if (cpus_have_cap(ARM64_WORKAROUND_NVIDIA_CARMEL_CNP)) |
1727 | return false; |
1728 | |
1729 | return has_cpuid_feature(entry, scope); |
1730 | } |
1731 | |
1732 | static bool __meltdown_safe = true; |
1733 | static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */ |
1734 | |
1735 | static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry, |
1736 | int scope) |
1737 | { |
1738 | /* List of CPUs that are not vulnerable and don't need KPTI */ |
1739 | static const struct midr_range kpti_safe_list[] = { |
1740 | MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2), |
1741 | MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN), |
1742 | MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53), |
1743 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A35), |
1744 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A53), |
1745 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A55), |
1746 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), |
1747 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), |
1748 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), |
1749 | MIDR_ALL_VERSIONS(MIDR_HISI_TSV110), |
1750 | MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), |
1751 | MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_GOLD), |
1752 | MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER), |
1753 | MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER), |
1754 | MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER), |
1755 | { /* sentinel */ } |
1756 | }; |
1757 | char const *str = "kpti command line option" ; |
1758 | bool meltdown_safe; |
1759 | |
1760 | meltdown_safe = is_midr_in_range_list(read_cpuid_id(), kpti_safe_list); |
1761 | |
1762 | /* Defer to CPU feature registers */ |
1763 | if (has_cpuid_feature(entry, scope)) |
1764 | meltdown_safe = true; |
1765 | |
1766 | if (!meltdown_safe) |
1767 | __meltdown_safe = false; |
1768 | |
1769 | /* |
1770 | * For reasons that aren't entirely clear, enabling KPTI on Cavium |
1771 | * ThunderX leads to apparent I-cache corruption of kernel text, which |
1772 | * ends as well as you might imagine. Don't even try. We cannot rely |
1773 | * on the cpus_have_*cap() helpers here to detect the CPU erratum |
1774 | * because cpucap detection order may change. However, since we know |
1775 | * affected CPUs are always in a homogeneous configuration, it is |
1776 | * safe to rely on this_cpu_has_cap() here. |
1777 | */ |
1778 | if (this_cpu_has_cap(ARM64_WORKAROUND_CAVIUM_27456)) { |
1779 | str = "ARM64_WORKAROUND_CAVIUM_27456" ; |
1780 | __kpti_forced = -1; |
1781 | } |
1782 | |
1783 | /* Useful for KASLR robustness */ |
1784 | if (kaslr_enabled() && kaslr_requires_kpti()) { |
1785 | if (!__kpti_forced) { |
1786 | str = "KASLR" ; |
1787 | __kpti_forced = 1; |
1788 | } |
1789 | } |
1790 | |
1791 | if (cpu_mitigations_off() && !__kpti_forced) { |
1792 | str = "mitigations=off" ; |
1793 | __kpti_forced = -1; |
1794 | } |
1795 | |
1796 | if (!IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) { |
1797 | pr_info_once("kernel page table isolation disabled by kernel configuration\n" ); |
1798 | return false; |
1799 | } |
1800 | |
1801 | /* Forced? */ |
1802 | if (__kpti_forced) { |
1803 | pr_info_once("kernel page table isolation forced %s by %s\n" , |
1804 | __kpti_forced > 0 ? "ON" : "OFF" , str); |
1805 | return __kpti_forced > 0; |
1806 | } |
1807 | |
1808 | return !meltdown_safe; |
1809 | } |
1810 | |
1811 | static bool has_nv1(const struct arm64_cpu_capabilities *entry, int scope) |
1812 | { |
1813 | /* |
1814 | * Although the Apple M2 family appears to support NV1, the |
1815 | * PTW barfs on the nVHE EL2 S1 page table format. Pretend |
1816 | * that it doesn't support NV1 at all. |
1817 | */ |
1818 | static const struct midr_range nv1_ni_list[] = { |
1819 | MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD), |
1820 | MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE), |
1821 | MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_PRO), |
1822 | MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_PRO), |
1823 | MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_MAX), |
1824 | MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_MAX), |
1825 | {} |
1826 | }; |
1827 | |
1828 | return (__system_matches_cap(ARM64_HAS_NESTED_VIRT) && |
1829 | !(has_cpuid_feature(entry, scope) || |
1830 | is_midr_in_range_list(read_cpuid_id(), nv1_ni_list))); |
1831 | } |
1832 | |
1833 | #if defined(ID_AA64MMFR0_EL1_TGRAN_LPA2) && defined(ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_LPA2) |
1834 | static bool has_lpa2_at_stage1(u64 mmfr0) |
1835 | { |
1836 | unsigned int tgran; |
1837 | |
1838 | tgran = cpuid_feature_extract_unsigned_field(mmfr0, |
1839 | ID_AA64MMFR0_EL1_TGRAN_SHIFT); |
1840 | return tgran == ID_AA64MMFR0_EL1_TGRAN_LPA2; |
1841 | } |
1842 | |
1843 | static bool has_lpa2_at_stage2(u64 mmfr0) |
1844 | { |
1845 | unsigned int tgran; |
1846 | |
1847 | tgran = cpuid_feature_extract_unsigned_field(mmfr0, |
1848 | ID_AA64MMFR0_EL1_TGRAN_2_SHIFT); |
1849 | return tgran == ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_LPA2; |
1850 | } |
1851 | |
1852 | static bool has_lpa2(const struct arm64_cpu_capabilities *entry, int scope) |
1853 | { |
1854 | u64 mmfr0; |
1855 | |
1856 | mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); |
1857 | return has_lpa2_at_stage1(mmfr0) && has_lpa2_at_stage2(mmfr0); |
1858 | } |
1859 | #else |
1860 | static bool has_lpa2(const struct arm64_cpu_capabilities *entry, int scope) |
1861 | { |
1862 | return false; |
1863 | } |
1864 | #endif |
1865 | |
1866 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
1867 | #define KPTI_NG_TEMP_VA (-(1UL << PMD_SHIFT)) |
1868 | |
1869 | extern |
1870 | void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, |
1871 | phys_addr_t size, pgprot_t prot, |
1872 | phys_addr_t (*pgtable_alloc)(int), int flags); |
1873 | |
1874 | static phys_addr_t __initdata kpti_ng_temp_alloc; |
1875 | |
1876 | static phys_addr_t __init kpti_ng_pgd_alloc(int shift) |
1877 | { |
1878 | kpti_ng_temp_alloc -= PAGE_SIZE; |
1879 | return kpti_ng_temp_alloc; |
1880 | } |
1881 | |
1882 | static int __init __kpti_install_ng_mappings(void *__unused) |
1883 | { |
1884 | typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long); |
1885 | extern kpti_remap_fn idmap_kpti_install_ng_mappings; |
1886 | kpti_remap_fn *remap_fn; |
1887 | |
1888 | int cpu = smp_processor_id(); |
1889 | int levels = CONFIG_PGTABLE_LEVELS; |
1890 | int order = order_base_2(levels); |
1891 | u64 kpti_ng_temp_pgd_pa = 0; |
1892 | pgd_t *kpti_ng_temp_pgd; |
1893 | u64 alloc = 0; |
1894 | |
1895 | if (levels == 5 && !pgtable_l5_enabled()) |
1896 | levels = 4; |
1897 | else if (levels == 4 && !pgtable_l4_enabled()) |
1898 | levels = 3; |
1899 | |
1900 | remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); |
1901 | |
1902 | if (!cpu) { |
1903 | alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order); |
1904 | kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE); |
1905 | kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd); |
1906 | |
1907 | // |
1908 | // Create a minimal page table hierarchy that permits us to map |
1909 | // the swapper page tables temporarily as we traverse them. |
1910 | // |
1911 | // The physical pages are laid out as follows: |
1912 | // |
1913 | // +--------+-/-------+-/------ +-/------ +-\\\--------+ |
1914 | // : PTE[] : | PMD[] : | PUD[] : | P4D[] : ||| PGD[] : |
1915 | // +--------+-\-------+-\------ +-\------ +-///--------+ |
1916 | // ^ |
1917 | // The first page is mapped into this hierarchy at a PMD_SHIFT |
1918 | // aligned virtual address, so that we can manipulate the PTE |
1919 | // level entries while the mapping is active. The first entry |
1920 | // covers the PTE[] page itself, the remaining entries are free |
1921 | // to be used as a ad-hoc fixmap. |
1922 | // |
1923 | create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc), |
1924 | KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL, |
1925 | kpti_ng_pgd_alloc, 0); |
1926 | } |
1927 | |
1928 | cpu_install_idmap(); |
1929 | remap_fn(cpu, num_online_cpus(), kpti_ng_temp_pgd_pa, KPTI_NG_TEMP_VA); |
1930 | cpu_uninstall_idmap(); |
1931 | |
1932 | if (!cpu) { |
1933 | free_pages(alloc, order); |
1934 | arm64_use_ng_mappings = true; |
1935 | } |
1936 | |
1937 | return 0; |
1938 | } |
1939 | |
1940 | static void __init kpti_install_ng_mappings(void) |
1941 | { |
1942 | /* Check whether KPTI is going to be used */ |
1943 | if (!arm64_kernel_unmapped_at_el0()) |
1944 | return; |
1945 | |
1946 | /* |
1947 | * We don't need to rewrite the page-tables if either we've done |
1948 | * it already or we have KASLR enabled and therefore have not |
1949 | * created any global mappings at all. |
1950 | */ |
1951 | if (arm64_use_ng_mappings) |
1952 | return; |
1953 | |
1954 | stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask); |
1955 | } |
1956 | |
1957 | #else |
1958 | static inline void kpti_install_ng_mappings(void) |
1959 | { |
1960 | } |
1961 | #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ |
1962 | |
1963 | static void cpu_enable_kpti(struct arm64_cpu_capabilities const *cap) |
1964 | { |
1965 | if (__this_cpu_read(this_cpu_vector) == vectors) { |
1966 | const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI); |
1967 | |
1968 | __this_cpu_write(this_cpu_vector, v); |
1969 | } |
1970 | |
1971 | } |
1972 | |
1973 | static int __init parse_kpti(char *str) |
1974 | { |
1975 | bool enabled; |
1976 | int ret = kstrtobool(s: str, res: &enabled); |
1977 | |
1978 | if (ret) |
1979 | return ret; |
1980 | |
1981 | __kpti_forced = enabled ? 1 : -1; |
1982 | return 0; |
1983 | } |
1984 | early_param("kpti" , parse_kpti); |
1985 | |
1986 | #ifdef CONFIG_ARM64_HW_AFDBM |
1987 | static struct cpumask dbm_cpus __read_mostly; |
1988 | |
1989 | static inline void __cpu_enable_hw_dbm(void) |
1990 | { |
1991 | u64 tcr = read_sysreg(tcr_el1) | TCR_HD; |
1992 | |
1993 | write_sysreg(tcr, tcr_el1); |
1994 | isb(); |
1995 | local_flush_tlb_all(); |
1996 | } |
1997 | |
1998 | static bool cpu_has_broken_dbm(void) |
1999 | { |
2000 | /* List of CPUs which have broken DBM support. */ |
2001 | static const struct midr_range cpus[] = { |
2002 | #ifdef CONFIG_ARM64_ERRATUM_1024718 |
2003 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A55), |
2004 | /* Kryo4xx Silver (rdpe => r1p0) */ |
2005 | MIDR_REV(MIDR_QCOM_KRYO_4XX_SILVER, 0xd, 0xe), |
2006 | #endif |
2007 | #ifdef CONFIG_ARM64_ERRATUM_2051678 |
2008 | MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2), |
2009 | #endif |
2010 | {}, |
2011 | }; |
2012 | |
2013 | return is_midr_in_range_list(read_cpuid_id(), cpus); |
2014 | } |
2015 | |
2016 | static bool cpu_can_use_dbm(const struct arm64_cpu_capabilities *cap) |
2017 | { |
2018 | return has_cpuid_feature(cap, SCOPE_LOCAL_CPU) && |
2019 | !cpu_has_broken_dbm(); |
2020 | } |
2021 | |
2022 | static void cpu_enable_hw_dbm(struct arm64_cpu_capabilities const *cap) |
2023 | { |
2024 | if (cpu_can_use_dbm(cap)) { |
2025 | __cpu_enable_hw_dbm(); |
2026 | cpumask_set_cpu(smp_processor_id(), &dbm_cpus); |
2027 | } |
2028 | } |
2029 | |
2030 | static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap, |
2031 | int __unused) |
2032 | { |
2033 | /* |
2034 | * DBM is a non-conflicting feature. i.e, the kernel can safely |
2035 | * run a mix of CPUs with and without the feature. So, we |
2036 | * unconditionally enable the capability to allow any late CPU |
2037 | * to use the feature. We only enable the control bits on the |
2038 | * CPU, if it is supported. |
2039 | */ |
2040 | |
2041 | return true; |
2042 | } |
2043 | |
2044 | #endif |
2045 | |
2046 | #ifdef CONFIG_ARM64_AMU_EXTN |
2047 | |
2048 | /* |
2049 | * The "amu_cpus" cpumask only signals that the CPU implementation for the |
2050 | * flagged CPUs supports the Activity Monitors Unit (AMU) but does not provide |
2051 | * information regarding all the events that it supports. When a CPU bit is |
2052 | * set in the cpumask, the user of this feature can only rely on the presence |
2053 | * of the 4 fixed counters for that CPU. But this does not guarantee that the |
2054 | * counters are enabled or access to these counters is enabled by code |
2055 | * executed at higher exception levels (firmware). |
2056 | */ |
2057 | static struct cpumask amu_cpus __read_mostly; |
2058 | |
2059 | bool cpu_has_amu_feat(int cpu) |
2060 | { |
2061 | return cpumask_test_cpu(cpu, &amu_cpus); |
2062 | } |
2063 | |
2064 | int get_cpu_with_amu_feat(void) |
2065 | { |
2066 | return cpumask_any(&amu_cpus); |
2067 | } |
2068 | |
2069 | static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap) |
2070 | { |
2071 | if (has_cpuid_feature(cap, SCOPE_LOCAL_CPU)) { |
2072 | cpumask_set_cpu(smp_processor_id(), &amu_cpus); |
2073 | |
2074 | /* 0 reference values signal broken/disabled counters */ |
2075 | if (!this_cpu_has_cap(ARM64_WORKAROUND_2457168)) |
2076 | update_freq_counters_refs(); |
2077 | } |
2078 | } |
2079 | |
2080 | static bool has_amu(const struct arm64_cpu_capabilities *cap, |
2081 | int __unused) |
2082 | { |
2083 | /* |
2084 | * The AMU extension is a non-conflicting feature: the kernel can |
2085 | * safely run a mix of CPUs with and without support for the |
2086 | * activity monitors extension. Therefore, unconditionally enable |
2087 | * the capability to allow any late CPU to use the feature. |
2088 | * |
2089 | * With this feature unconditionally enabled, the cpu_enable |
2090 | * function will be called for all CPUs that match the criteria, |
2091 | * including secondary and hotplugged, marking this feature as |
2092 | * present on that respective CPU. The enable function will also |
2093 | * print a detection message. |
2094 | */ |
2095 | |
2096 | return true; |
2097 | } |
2098 | #else |
2099 | int get_cpu_with_amu_feat(void) |
2100 | { |
2101 | return nr_cpu_ids; |
2102 | } |
2103 | #endif |
2104 | |
2105 | static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused) |
2106 | { |
2107 | return is_kernel_in_hyp_mode(); |
2108 | } |
2109 | |
2110 | static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused) |
2111 | { |
2112 | /* |
2113 | * Copy register values that aren't redirected by hardware. |
2114 | * |
2115 | * Before code patching, we only set tpidr_el1, all CPUs need to copy |
2116 | * this value to tpidr_el2 before we patch the code. Once we've done |
2117 | * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to |
2118 | * do anything here. |
2119 | */ |
2120 | if (!alternative_is_applied(ARM64_HAS_VIRT_HOST_EXTN)) |
2121 | write_sysreg(read_sysreg(tpidr_el1), tpidr_el2); |
2122 | } |
2123 | |
2124 | static bool has_nested_virt_support(const struct arm64_cpu_capabilities *cap, |
2125 | int scope) |
2126 | { |
2127 | if (kvm_get_mode() != KVM_MODE_NV) |
2128 | return false; |
2129 | |
2130 | if (!has_cpuid_feature(entry: cap, scope)) { |
2131 | pr_warn("unavailable: %s\n" , cap->desc); |
2132 | return false; |
2133 | } |
2134 | |
2135 | return true; |
2136 | } |
2137 | |
2138 | static bool hvhe_possible(const struct arm64_cpu_capabilities *entry, |
2139 | int __unused) |
2140 | { |
2141 | return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE); |
2142 | } |
2143 | |
2144 | #ifdef CONFIG_ARM64_PAN |
2145 | static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) |
2146 | { |
2147 | /* |
2148 | * We modify PSTATE. This won't work from irq context as the PSTATE |
2149 | * is discarded once we return from the exception. |
2150 | */ |
2151 | WARN_ON_ONCE(in_interrupt()); |
2152 | |
2153 | sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0); |
2154 | set_pstate_pan(1); |
2155 | } |
2156 | #endif /* CONFIG_ARM64_PAN */ |
2157 | |
2158 | #ifdef CONFIG_ARM64_RAS_EXTN |
2159 | static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused) |
2160 | { |
2161 | /* Firmware may have left a deferred SError in this register. */ |
2162 | write_sysreg_s(0, SYS_DISR_EL1); |
2163 | } |
2164 | #endif /* CONFIG_ARM64_RAS_EXTN */ |
2165 | |
2166 | #ifdef CONFIG_ARM64_PTR_AUTH |
2167 | static bool has_address_auth_cpucap(const struct arm64_cpu_capabilities *entry, int scope) |
2168 | { |
2169 | int boot_val, sec_val; |
2170 | |
2171 | /* We don't expect to be called with SCOPE_SYSTEM */ |
2172 | WARN_ON(scope == SCOPE_SYSTEM); |
2173 | /* |
2174 | * The ptr-auth feature levels are not intercompatible with lower |
2175 | * levels. Hence we must match ptr-auth feature level of the secondary |
2176 | * CPUs with that of the boot CPU. The level of boot cpu is fetched |
2177 | * from the sanitised register whereas direct register read is done for |
2178 | * the secondary CPUs. |
2179 | * The sanitised feature state is guaranteed to match that of the |
2180 | * boot CPU as a mismatched secondary CPU is parked before it gets |
2181 | * a chance to update the state, with the capability. |
2182 | */ |
2183 | boot_val = cpuid_feature_extract_field(read_sanitised_ftr_reg(entry->sys_reg), |
2184 | entry->field_pos, entry->sign); |
2185 | if (scope & SCOPE_BOOT_CPU) |
2186 | return boot_val >= entry->min_field_value; |
2187 | /* Now check for the secondary CPUs with SCOPE_LOCAL_CPU scope */ |
2188 | sec_val = cpuid_feature_extract_field(__read_sysreg_by_encoding(entry->sys_reg), |
2189 | entry->field_pos, entry->sign); |
2190 | return (sec_val >= entry->min_field_value) && (sec_val == boot_val); |
2191 | } |
2192 | |
2193 | static bool has_address_auth_metacap(const struct arm64_cpu_capabilities *entry, |
2194 | int scope) |
2195 | { |
2196 | bool api = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_IMP_DEF], scope); |
2197 | bool apa = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5], scope); |
2198 | bool apa3 = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3], scope); |
2199 | |
2200 | return apa || apa3 || api; |
2201 | } |
2202 | |
2203 | static bool has_generic_auth(const struct arm64_cpu_capabilities *entry, |
2204 | int __unused) |
2205 | { |
2206 | bool gpi = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF); |
2207 | bool gpa = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH_QARMA5); |
2208 | bool gpa3 = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH_QARMA3); |
2209 | |
2210 | return gpa || gpa3 || gpi; |
2211 | } |
2212 | #endif /* CONFIG_ARM64_PTR_AUTH */ |
2213 | |
2214 | #ifdef CONFIG_ARM64_E0PD |
2215 | static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap) |
2216 | { |
2217 | if (this_cpu_has_cap(ARM64_HAS_E0PD)) |
2218 | sysreg_clear_set(tcr_el1, 0, TCR_E0PD1); |
2219 | } |
2220 | #endif /* CONFIG_ARM64_E0PD */ |
2221 | |
2222 | #ifdef CONFIG_ARM64_PSEUDO_NMI |
2223 | static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry, |
2224 | int scope) |
2225 | { |
2226 | /* |
2227 | * ARM64_HAS_GIC_CPUIF_SYSREGS has a lower index, and is a boot CPU |
2228 | * feature, so will be detected earlier. |
2229 | */ |
2230 | BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_MASKING <= ARM64_HAS_GIC_CPUIF_SYSREGS); |
2231 | if (!cpus_have_cap(ARM64_HAS_GIC_CPUIF_SYSREGS)) |
2232 | return false; |
2233 | |
2234 | return enable_pseudo_nmi; |
2235 | } |
2236 | |
2237 | static bool has_gic_prio_relaxed_sync(const struct arm64_cpu_capabilities *entry, |
2238 | int scope) |
2239 | { |
2240 | /* |
2241 | * If we're not using priority masking then we won't be poking PMR_EL1, |
2242 | * and there's no need to relax synchronization of writes to it, and |
2243 | * ICC_CTLR_EL1 might not be accessible and we must avoid reads from |
2244 | * that. |
2245 | * |
2246 | * ARM64_HAS_GIC_PRIO_MASKING has a lower index, and is a boot CPU |
2247 | * feature, so will be detected earlier. |
2248 | */ |
2249 | BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_RELAXED_SYNC <= ARM64_HAS_GIC_PRIO_MASKING); |
2250 | if (!cpus_have_cap(ARM64_HAS_GIC_PRIO_MASKING)) |
2251 | return false; |
2252 | |
2253 | /* |
2254 | * When Priority Mask Hint Enable (PMHE) == 0b0, PMR is not used as a |
2255 | * hint for interrupt distribution, a DSB is not necessary when |
2256 | * unmasking IRQs via PMR, and we can relax the barrier to a NOP. |
2257 | * |
2258 | * Linux itself doesn't use 1:N distribution, so has no need to |
2259 | * set PMHE. The only reason to have it set is if EL3 requires it |
2260 | * (and we can't change it). |
2261 | */ |
2262 | return (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) == 0; |
2263 | } |
2264 | #endif |
2265 | |
2266 | #ifdef CONFIG_ARM64_BTI |
2267 | static void bti_enable(const struct arm64_cpu_capabilities *__unused) |
2268 | { |
2269 | /* |
2270 | * Use of X16/X17 for tail-calls and trampolines that jump to |
2271 | * function entry points using BR is a requirement for |
2272 | * marking binaries with GNU_PROPERTY_AARCH64_FEATURE_1_BTI. |
2273 | * So, be strict and forbid other BRs using other registers to |
2274 | * jump onto a PACIxSP instruction: |
2275 | */ |
2276 | sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1); |
2277 | isb(); |
2278 | } |
2279 | #endif /* CONFIG_ARM64_BTI */ |
2280 | |
2281 | #ifdef CONFIG_ARM64_MTE |
2282 | static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) |
2283 | { |
2284 | sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0); |
2285 | |
2286 | mte_cpu_setup(); |
2287 | |
2288 | /* |
2289 | * Clear the tags in the zero page. This needs to be done via the |
2290 | * linear map which has the Tagged attribute. |
2291 | */ |
2292 | if (try_page_mte_tagging(ZERO_PAGE(0))) { |
2293 | mte_clear_page_tags(lm_alias(empty_zero_page)); |
2294 | set_page_mte_tagged(ZERO_PAGE(0)); |
2295 | } |
2296 | |
2297 | kasan_init_hw_tags_cpu(); |
2298 | } |
2299 | #endif /* CONFIG_ARM64_MTE */ |
2300 | |
2301 | static void user_feature_fixup(void) |
2302 | { |
2303 | if (cpus_have_cap(ARM64_WORKAROUND_2658417)) { |
2304 | struct arm64_ftr_reg *regp; |
2305 | |
2306 | regp = get_arm64_ftr_reg(SYS_ID_AA64ISAR1_EL1); |
2307 | if (regp) |
2308 | regp->user_mask &= ~ID_AA64ISAR1_EL1_BF16_MASK; |
2309 | } |
2310 | } |
2311 | |
2312 | static void elf_hwcap_fixup(void) |
2313 | { |
2314 | #ifdef CONFIG_COMPAT |
2315 | if (cpus_have_cap(ARM64_WORKAROUND_1742098)) |
2316 | compat_elf_hwcap2 &= ~COMPAT_HWCAP2_AES; |
2317 | #endif /* CONFIG_COMPAT */ |
2318 | } |
2319 | |
2320 | #ifdef CONFIG_KVM |
2321 | static bool is_kvm_protected_mode(const struct arm64_cpu_capabilities *entry, int __unused) |
2322 | { |
2323 | return kvm_get_mode() == KVM_MODE_PROTECTED; |
2324 | } |
2325 | #endif /* CONFIG_KVM */ |
2326 | |
2327 | static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused) |
2328 | { |
2329 | sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP); |
2330 | } |
2331 | |
2332 | static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) |
2333 | { |
2334 | set_pstate_dit(1); |
2335 | } |
2336 | |
2337 | static void cpu_enable_mops(const struct arm64_cpu_capabilities *__unused) |
2338 | { |
2339 | sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_MSCEn); |
2340 | } |
2341 | |
2342 | /* Internal helper functions to match cpu capability type */ |
2343 | static bool |
2344 | cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap) |
2345 | { |
2346 | return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU); |
2347 | } |
2348 | |
2349 | static bool |
2350 | cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap) |
2351 | { |
2352 | return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU); |
2353 | } |
2354 | |
2355 | static bool |
2356 | cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap) |
2357 | { |
2358 | return !!(cap->type & ARM64_CPUCAP_PANIC_ON_CONFLICT); |
2359 | } |
2360 | |
2361 | static const struct arm64_cpu_capabilities arm64_features[] = { |
2362 | { |
2363 | .capability = ARM64_ALWAYS_BOOT, |
2364 | .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, |
2365 | .matches = has_always, |
2366 | }, |
2367 | { |
2368 | .capability = ARM64_ALWAYS_SYSTEM, |
2369 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2370 | .matches = has_always, |
2371 | }, |
2372 | { |
2373 | .desc = "GIC system register CPU interface" , |
2374 | .capability = ARM64_HAS_GIC_CPUIF_SYSREGS, |
2375 | .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, |
2376 | .matches = has_useable_gicv3_cpuif, |
2377 | ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, GIC, IMP) |
2378 | }, |
2379 | { |
2380 | .desc = "Enhanced Counter Virtualization" , |
2381 | .capability = ARM64_HAS_ECV, |
2382 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2383 | .matches = has_cpuid_feature, |
2384 | ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, ECV, IMP) |
2385 | }, |
2386 | { |
2387 | .desc = "Enhanced Counter Virtualization (CNTPOFF)" , |
2388 | .capability = ARM64_HAS_ECV_CNTPOFF, |
2389 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2390 | .matches = has_cpuid_feature, |
2391 | ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, ECV, CNTPOFF) |
2392 | }, |
2393 | #ifdef CONFIG_ARM64_PAN |
2394 | { |
2395 | .desc = "Privileged Access Never" , |
2396 | .capability = ARM64_HAS_PAN, |
2397 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2398 | .matches = has_cpuid_feature, |
2399 | .cpu_enable = cpu_enable_pan, |
2400 | ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, PAN, IMP) |
2401 | }, |
2402 | #endif /* CONFIG_ARM64_PAN */ |
2403 | #ifdef CONFIG_ARM64_EPAN |
2404 | { |
2405 | .desc = "Enhanced Privileged Access Never" , |
2406 | .capability = ARM64_HAS_EPAN, |
2407 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2408 | .matches = has_cpuid_feature, |
2409 | ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, PAN, PAN3) |
2410 | }, |
2411 | #endif /* CONFIG_ARM64_EPAN */ |
2412 | #ifdef CONFIG_ARM64_LSE_ATOMICS |
2413 | { |
2414 | .desc = "LSE atomic instructions" , |
2415 | .capability = ARM64_HAS_LSE_ATOMICS, |
2416 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2417 | .matches = has_cpuid_feature, |
2418 | ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, ATOMIC, IMP) |
2419 | }, |
2420 | #endif /* CONFIG_ARM64_LSE_ATOMICS */ |
2421 | { |
2422 | .desc = "Virtualization Host Extensions" , |
2423 | .capability = ARM64_HAS_VIRT_HOST_EXTN, |
2424 | .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, |
2425 | .matches = runs_at_el2, |
2426 | .cpu_enable = cpu_copy_el2regs, |
2427 | }, |
2428 | { |
2429 | .desc = "Nested Virtualization Support" , |
2430 | .capability = ARM64_HAS_NESTED_VIRT, |
2431 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2432 | .matches = has_nested_virt_support, |
2433 | ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, NV, NV2) |
2434 | }, |
2435 | { |
2436 | .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE, |
2437 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2438 | .matches = has_32bit_el0, |
2439 | ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, EL0, AARCH32) |
2440 | }, |
2441 | #ifdef CONFIG_KVM |
2442 | { |
2443 | .desc = "32-bit EL1 Support" , |
2444 | .capability = ARM64_HAS_32BIT_EL1, |
2445 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2446 | .matches = has_cpuid_feature, |
2447 | ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, EL1, AARCH32) |
2448 | }, |
2449 | { |
2450 | .desc = "Protected KVM" , |
2451 | .capability = ARM64_KVM_PROTECTED_MODE, |
2452 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2453 | .matches = is_kvm_protected_mode, |
2454 | }, |
2455 | { |
2456 | .desc = "HCRX_EL2 register" , |
2457 | .capability = ARM64_HAS_HCX, |
2458 | .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, |
2459 | .matches = has_cpuid_feature, |
2460 | ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HCX, IMP) |
2461 | }, |
2462 | #endif |
2463 | { |
2464 | .desc = "Kernel page table isolation (KPTI)" , |
2465 | .capability = ARM64_UNMAP_KERNEL_AT_EL0, |
2466 | .type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE, |
2467 | .cpu_enable = cpu_enable_kpti, |
2468 | .matches = unmap_kernel_at_el0, |
2469 | /* |
2470 | * The ID feature fields below are used to indicate that |
2471 | * the CPU doesn't need KPTI. See unmap_kernel_at_el0 for |
2472 | * more details. |
2473 | */ |
2474 | ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, CSV3, IMP) |
2475 | }, |
2476 | { |
2477 | .capability = ARM64_HAS_FPSIMD, |
2478 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2479 | .matches = has_cpuid_feature, |
2480 | .cpu_enable = cpu_enable_fpsimd, |
2481 | ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, FP, IMP) |
2482 | }, |
2483 | #ifdef CONFIG_ARM64_PMEM |
2484 | { |
2485 | .desc = "Data cache clean to Point of Persistence" , |
2486 | .capability = ARM64_HAS_DCPOP, |
2487 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2488 | .matches = has_cpuid_feature, |
2489 | ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, DPB, IMP) |
2490 | }, |
2491 | { |
2492 | .desc = "Data cache clean to Point of Deep Persistence" , |
2493 | .capability = ARM64_HAS_DCPODP, |
2494 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2495 | .matches = has_cpuid_feature, |
2496 | ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, DPB, DPB2) |
2497 | }, |
2498 | #endif |
2499 | #ifdef CONFIG_ARM64_SVE |
2500 | { |
2501 | .desc = "Scalable Vector Extension" , |
2502 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2503 | .capability = ARM64_SVE, |
2504 | .cpu_enable = cpu_enable_sve, |
2505 | .matches = has_cpuid_feature, |
2506 | ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, SVE, IMP) |
2507 | }, |
2508 | #endif /* CONFIG_ARM64_SVE */ |
2509 | #ifdef CONFIG_ARM64_RAS_EXTN |
2510 | { |
2511 | .desc = "RAS Extension Support" , |
2512 | .capability = ARM64_HAS_RAS_EXTN, |
2513 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2514 | .matches = has_cpuid_feature, |
2515 | .cpu_enable = cpu_clear_disr, |
2516 | ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, RAS, IMP) |
2517 | }, |
2518 | #endif /* CONFIG_ARM64_RAS_EXTN */ |
2519 | #ifdef CONFIG_ARM64_AMU_EXTN |
2520 | { |
2521 | .desc = "Activity Monitors Unit (AMU)" , |
2522 | .capability = ARM64_HAS_AMU_EXTN, |
2523 | .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, |
2524 | .matches = has_amu, |
2525 | .cpu_enable = cpu_amu_enable, |
2526 | .cpus = &amu_cpus, |
2527 | ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, AMU, IMP) |
2528 | }, |
2529 | #endif /* CONFIG_ARM64_AMU_EXTN */ |
2530 | { |
2531 | .desc = "Data cache clean to the PoU not required for I/D coherence" , |
2532 | .capability = ARM64_HAS_CACHE_IDC, |
2533 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2534 | .matches = has_cache_idc, |
2535 | .cpu_enable = cpu_emulate_effective_ctr, |
2536 | }, |
2537 | { |
2538 | .desc = "Instruction cache invalidation not required for I/D coherence" , |
2539 | .capability = ARM64_HAS_CACHE_DIC, |
2540 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2541 | .matches = has_cache_dic, |
2542 | }, |
2543 | { |
2544 | .desc = "Stage-2 Force Write-Back" , |
2545 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2546 | .capability = ARM64_HAS_STAGE2_FWB, |
2547 | .matches = has_cpuid_feature, |
2548 | ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, FWB, IMP) |
2549 | }, |
2550 | { |
2551 | .desc = "ARMv8.4 Translation Table Level" , |
2552 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2553 | .capability = ARM64_HAS_ARMv8_4_TTL, |
2554 | .matches = has_cpuid_feature, |
2555 | ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, TTL, IMP) |
2556 | }, |
2557 | { |
2558 | .desc = "TLB range maintenance instructions" , |
2559 | .capability = ARM64_HAS_TLB_RANGE, |
2560 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2561 | .matches = has_cpuid_feature, |
2562 | ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, TLB, RANGE) |
2563 | }, |
2564 | #ifdef CONFIG_ARM64_HW_AFDBM |
2565 | { |
2566 | .desc = "Hardware dirty bit management" , |
2567 | .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, |
2568 | .capability = ARM64_HW_DBM, |
2569 | .matches = has_hw_dbm, |
2570 | .cpu_enable = cpu_enable_hw_dbm, |
2571 | .cpus = &dbm_cpus, |
2572 | ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, DBM) |
2573 | }, |
2574 | #endif |
2575 | { |
2576 | .desc = "CRC32 instructions" , |
2577 | .capability = ARM64_HAS_CRC32, |
2578 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2579 | .matches = has_cpuid_feature, |
2580 | ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, CRC32, IMP) |
2581 | }, |
2582 | { |
2583 | .desc = "Speculative Store Bypassing Safe (SSBS)" , |
2584 | .capability = ARM64_SSBS, |
2585 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2586 | .matches = has_cpuid_feature, |
2587 | ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SSBS, IMP) |
2588 | }, |
2589 | #ifdef CONFIG_ARM64_CNP |
2590 | { |
2591 | .desc = "Common not Private translations" , |
2592 | .capability = ARM64_HAS_CNP, |
2593 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2594 | .matches = has_useable_cnp, |
2595 | .cpu_enable = cpu_enable_cnp, |
2596 | ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, CnP, IMP) |
2597 | }, |
2598 | #endif |
2599 | { |
2600 | .desc = "Speculation barrier (SB)" , |
2601 | .capability = ARM64_HAS_SB, |
2602 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2603 | .matches = has_cpuid_feature, |
2604 | ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, SB, IMP) |
2605 | }, |
2606 | #ifdef CONFIG_ARM64_PTR_AUTH |
2607 | { |
2608 | .desc = "Address authentication (architected QARMA5 algorithm)" , |
2609 | .capability = ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5, |
2610 | .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, |
2611 | .matches = has_address_auth_cpucap, |
2612 | ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, APA, PAuth) |
2613 | }, |
2614 | { |
2615 | .desc = "Address authentication (architected QARMA3 algorithm)" , |
2616 | .capability = ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3, |
2617 | .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, |
2618 | .matches = has_address_auth_cpucap, |
2619 | ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, APA3, PAuth) |
2620 | }, |
2621 | { |
2622 | .desc = "Address authentication (IMP DEF algorithm)" , |
2623 | .capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF, |
2624 | .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, |
2625 | .matches = has_address_auth_cpucap, |
2626 | ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, API, PAuth) |
2627 | }, |
2628 | { |
2629 | .capability = ARM64_HAS_ADDRESS_AUTH, |
2630 | .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, |
2631 | .matches = has_address_auth_metacap, |
2632 | }, |
2633 | { |
2634 | .desc = "Generic authentication (architected QARMA5 algorithm)" , |
2635 | .capability = ARM64_HAS_GENERIC_AUTH_ARCH_QARMA5, |
2636 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2637 | .matches = has_cpuid_feature, |
2638 | ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, GPA, IMP) |
2639 | }, |
2640 | { |
2641 | .desc = "Generic authentication (architected QARMA3 algorithm)" , |
2642 | .capability = ARM64_HAS_GENERIC_AUTH_ARCH_QARMA3, |
2643 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2644 | .matches = has_cpuid_feature, |
2645 | ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, GPA3, IMP) |
2646 | }, |
2647 | { |
2648 | .desc = "Generic authentication (IMP DEF algorithm)" , |
2649 | .capability = ARM64_HAS_GENERIC_AUTH_IMP_DEF, |
2650 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2651 | .matches = has_cpuid_feature, |
2652 | ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, GPI, IMP) |
2653 | }, |
2654 | { |
2655 | .capability = ARM64_HAS_GENERIC_AUTH, |
2656 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2657 | .matches = has_generic_auth, |
2658 | }, |
2659 | #endif /* CONFIG_ARM64_PTR_AUTH */ |
2660 | #ifdef CONFIG_ARM64_PSEUDO_NMI |
2661 | { |
2662 | /* |
2663 | * Depends on having GICv3 |
2664 | */ |
2665 | .desc = "IRQ priority masking" , |
2666 | .capability = ARM64_HAS_GIC_PRIO_MASKING, |
2667 | .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, |
2668 | .matches = can_use_gic_priorities, |
2669 | }, |
2670 | { |
2671 | /* |
2672 | * Depends on ARM64_HAS_GIC_PRIO_MASKING |
2673 | */ |
2674 | .capability = ARM64_HAS_GIC_PRIO_RELAXED_SYNC, |
2675 | .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, |
2676 | .matches = has_gic_prio_relaxed_sync, |
2677 | }, |
2678 | #endif |
2679 | #ifdef CONFIG_ARM64_E0PD |
2680 | { |
2681 | .desc = "E0PD" , |
2682 | .capability = ARM64_HAS_E0PD, |
2683 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2684 | .cpu_enable = cpu_enable_e0pd, |
2685 | .matches = has_cpuid_feature, |
2686 | ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, E0PD, IMP) |
2687 | }, |
2688 | #endif |
2689 | { |
2690 | .desc = "Random Number Generator" , |
2691 | .capability = ARM64_HAS_RNG, |
2692 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2693 | .matches = has_cpuid_feature, |
2694 | ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, RNDR, IMP) |
2695 | }, |
2696 | #ifdef CONFIG_ARM64_BTI |
2697 | { |
2698 | .desc = "Branch Target Identification" , |
2699 | .capability = ARM64_BTI, |
2700 | #ifdef CONFIG_ARM64_BTI_KERNEL |
2701 | .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, |
2702 | #else |
2703 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2704 | #endif |
2705 | .matches = has_cpuid_feature, |
2706 | .cpu_enable = bti_enable, |
2707 | ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, BT, IMP) |
2708 | }, |
2709 | #endif |
2710 | #ifdef CONFIG_ARM64_MTE |
2711 | { |
2712 | .desc = "Memory Tagging Extension" , |
2713 | .capability = ARM64_MTE, |
2714 | .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, |
2715 | .matches = has_cpuid_feature, |
2716 | .cpu_enable = cpu_enable_mte, |
2717 | ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, MTE, MTE2) |
2718 | }, |
2719 | { |
2720 | .desc = "Asymmetric MTE Tag Check Fault" , |
2721 | .capability = ARM64_MTE_ASYMM, |
2722 | .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, |
2723 | .matches = has_cpuid_feature, |
2724 | ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, MTE, MTE3) |
2725 | }, |
2726 | #endif /* CONFIG_ARM64_MTE */ |
2727 | { |
2728 | .desc = "RCpc load-acquire (LDAPR)" , |
2729 | .capability = ARM64_HAS_LDAPR, |
2730 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2731 | .matches = has_cpuid_feature, |
2732 | ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, LRCPC, IMP) |
2733 | }, |
2734 | { |
2735 | .desc = "Fine Grained Traps" , |
2736 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2737 | .capability = ARM64_HAS_FGT, |
2738 | .matches = has_cpuid_feature, |
2739 | ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, FGT, IMP) |
2740 | }, |
2741 | #ifdef CONFIG_ARM64_SME |
2742 | { |
2743 | .desc = "Scalable Matrix Extension" , |
2744 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2745 | .capability = ARM64_SME, |
2746 | .matches = has_cpuid_feature, |
2747 | .cpu_enable = cpu_enable_sme, |
2748 | ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SME, IMP) |
2749 | }, |
2750 | /* FA64 should be sorted after the base SME capability */ |
2751 | { |
2752 | .desc = "FA64" , |
2753 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2754 | .capability = ARM64_SME_FA64, |
2755 | .matches = has_cpuid_feature, |
2756 | .cpu_enable = cpu_enable_fa64, |
2757 | ARM64_CPUID_FIELDS(ID_AA64SMFR0_EL1, FA64, IMP) |
2758 | }, |
2759 | { |
2760 | .desc = "SME2" , |
2761 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2762 | .capability = ARM64_SME2, |
2763 | .matches = has_cpuid_feature, |
2764 | .cpu_enable = cpu_enable_sme2, |
2765 | ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SME, SME2) |
2766 | }, |
2767 | #endif /* CONFIG_ARM64_SME */ |
2768 | { |
2769 | .desc = "WFx with timeout" , |
2770 | .capability = ARM64_HAS_WFXT, |
2771 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2772 | .matches = has_cpuid_feature, |
2773 | ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, WFxT, IMP) |
2774 | }, |
2775 | { |
2776 | .desc = "Trap EL0 IMPLEMENTATION DEFINED functionality" , |
2777 | .capability = ARM64_HAS_TIDCP1, |
2778 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2779 | .matches = has_cpuid_feature, |
2780 | .cpu_enable = cpu_trap_el0_impdef, |
2781 | ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, TIDCP1, IMP) |
2782 | }, |
2783 | { |
2784 | .desc = "Data independent timing control (DIT)" , |
2785 | .capability = ARM64_HAS_DIT, |
2786 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2787 | .matches = has_cpuid_feature, |
2788 | .cpu_enable = cpu_enable_dit, |
2789 | ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, DIT, IMP) |
2790 | }, |
2791 | { |
2792 | .desc = "Memory Copy and Memory Set instructions" , |
2793 | .capability = ARM64_HAS_MOPS, |
2794 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2795 | .matches = has_cpuid_feature, |
2796 | .cpu_enable = cpu_enable_mops, |
2797 | ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, MOPS, IMP) |
2798 | }, |
2799 | { |
2800 | .capability = ARM64_HAS_TCR2, |
2801 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2802 | .matches = has_cpuid_feature, |
2803 | ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, TCRX, IMP) |
2804 | }, |
2805 | { |
2806 | .desc = "Stage-1 Permission Indirection Extension (S1PIE)" , |
2807 | .capability = ARM64_HAS_S1PIE, |
2808 | .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, |
2809 | .matches = has_cpuid_feature, |
2810 | ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, S1PIE, IMP) |
2811 | }, |
2812 | { |
2813 | .desc = "VHE for hypervisor only" , |
2814 | .capability = ARM64_KVM_HVHE, |
2815 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2816 | .matches = hvhe_possible, |
2817 | }, |
2818 | { |
2819 | .desc = "Enhanced Virtualization Traps" , |
2820 | .capability = ARM64_HAS_EVT, |
2821 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2822 | .matches = has_cpuid_feature, |
2823 | ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, EVT, IMP) |
2824 | }, |
2825 | { |
2826 | .desc = "52-bit Virtual Addressing for KVM (LPA2)" , |
2827 | .capability = ARM64_HAS_LPA2, |
2828 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2829 | .matches = has_lpa2, |
2830 | }, |
2831 | { |
2832 | .desc = "FPMR" , |
2833 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2834 | .capability = ARM64_HAS_FPMR, |
2835 | .matches = has_cpuid_feature, |
2836 | .cpu_enable = cpu_enable_fpmr, |
2837 | ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, FPMR, IMP) |
2838 | }, |
2839 | #ifdef CONFIG_ARM64_VA_BITS_52 |
2840 | { |
2841 | .capability = ARM64_HAS_VA52, |
2842 | .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, |
2843 | .matches = has_cpuid_feature, |
2844 | #ifdef CONFIG_ARM64_64K_PAGES |
2845 | .desc = "52-bit Virtual Addressing (LVA)" , |
2846 | ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, VARange, 52) |
2847 | #else |
2848 | .desc = "52-bit Virtual Addressing (LPA2)" , |
2849 | #ifdef CONFIG_ARM64_4K_PAGES |
2850 | ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN4, 52_BIT) |
2851 | #else |
2852 | ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN16, 52_BIT) |
2853 | #endif |
2854 | #endif |
2855 | }, |
2856 | #endif |
2857 | { |
2858 | .desc = "NV1" , |
2859 | .capability = ARM64_HAS_HCR_NV1, |
2860 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2861 | .matches = has_nv1, |
2862 | ARM64_CPUID_FIELDS_NEG(ID_AA64MMFR4_EL1, E2H0, NI_NV1) |
2863 | }, |
2864 | {}, |
2865 | }; |
2866 | |
2867 | #define HWCAP_CPUID_MATCH(reg, field, min_value) \ |
2868 | .matches = has_user_cpuid_feature, \ |
2869 | ARM64_CPUID_FIELDS(reg, field, min_value) |
2870 | |
2871 | #define __HWCAP_CAP(name, cap_type, cap) \ |
2872 | .desc = name, \ |
2873 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, \ |
2874 | .hwcap_type = cap_type, \ |
2875 | .hwcap = cap, \ |
2876 | |
2877 | #define HWCAP_CAP(reg, field, min_value, cap_type, cap) \ |
2878 | { \ |
2879 | __HWCAP_CAP(#cap, cap_type, cap) \ |
2880 | HWCAP_CPUID_MATCH(reg, field, min_value) \ |
2881 | } |
2882 | |
2883 | #define HWCAP_MULTI_CAP(list, cap_type, cap) \ |
2884 | { \ |
2885 | __HWCAP_CAP(#cap, cap_type, cap) \ |
2886 | .matches = cpucap_multi_entry_cap_matches, \ |
2887 | .match_list = list, \ |
2888 | } |
2889 | |
2890 | #define HWCAP_CAP_MATCH(match, cap_type, cap) \ |
2891 | { \ |
2892 | __HWCAP_CAP(#cap, cap_type, cap) \ |
2893 | .matches = match, \ |
2894 | } |
2895 | |
2896 | #ifdef CONFIG_ARM64_PTR_AUTH |
2897 | static const struct arm64_cpu_capabilities ptr_auth_hwcap_addr_matches[] = { |
2898 | { |
2899 | HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, APA, PAuth) |
2900 | }, |
2901 | { |
2902 | HWCAP_CPUID_MATCH(ID_AA64ISAR2_EL1, APA3, PAuth) |
2903 | }, |
2904 | { |
2905 | HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, API, PAuth) |
2906 | }, |
2907 | {}, |
2908 | }; |
2909 | |
2910 | static const struct arm64_cpu_capabilities ptr_auth_hwcap_gen_matches[] = { |
2911 | { |
2912 | HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, GPA, IMP) |
2913 | }, |
2914 | { |
2915 | HWCAP_CPUID_MATCH(ID_AA64ISAR2_EL1, GPA3, IMP) |
2916 | }, |
2917 | { |
2918 | HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, GPI, IMP) |
2919 | }, |
2920 | {}, |
2921 | }; |
2922 | #endif |
2923 | |
2924 | static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { |
2925 | HWCAP_CAP(ID_AA64ISAR0_EL1, AES, PMULL, CAP_HWCAP, KERNEL_HWCAP_PMULL), |
2926 | HWCAP_CAP(ID_AA64ISAR0_EL1, AES, AES, CAP_HWCAP, KERNEL_HWCAP_AES), |
2927 | HWCAP_CAP(ID_AA64ISAR0_EL1, SHA1, IMP, CAP_HWCAP, KERNEL_HWCAP_SHA1), |
2928 | HWCAP_CAP(ID_AA64ISAR0_EL1, SHA2, SHA256, CAP_HWCAP, KERNEL_HWCAP_SHA2), |
2929 | HWCAP_CAP(ID_AA64ISAR0_EL1, SHA2, SHA512, CAP_HWCAP, KERNEL_HWCAP_SHA512), |
2930 | HWCAP_CAP(ID_AA64ISAR0_EL1, CRC32, IMP, CAP_HWCAP, KERNEL_HWCAP_CRC32), |
2931 | HWCAP_CAP(ID_AA64ISAR0_EL1, ATOMIC, IMP, CAP_HWCAP, KERNEL_HWCAP_ATOMICS), |
2932 | HWCAP_CAP(ID_AA64ISAR0_EL1, ATOMIC, FEAT_LSE128, CAP_HWCAP, KERNEL_HWCAP_LSE128), |
2933 | HWCAP_CAP(ID_AA64ISAR0_EL1, RDM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDRDM), |
2934 | HWCAP_CAP(ID_AA64ISAR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SHA3), |
2935 | HWCAP_CAP(ID_AA64ISAR0_EL1, SM3, IMP, CAP_HWCAP, KERNEL_HWCAP_SM3), |
2936 | HWCAP_CAP(ID_AA64ISAR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SM4), |
2937 | HWCAP_CAP(ID_AA64ISAR0_EL1, DP, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP), |
2938 | HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM), |
2939 | HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM, CAP_HWCAP, KERNEL_HWCAP_FLAGM), |
2940 | HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2), |
2941 | HWCAP_CAP(ID_AA64ISAR0_EL1, RNDR, IMP, CAP_HWCAP, KERNEL_HWCAP_RNG), |
2942 | HWCAP_CAP(ID_AA64PFR0_EL1, FP, IMP, CAP_HWCAP, KERNEL_HWCAP_FP), |
2943 | HWCAP_CAP(ID_AA64PFR0_EL1, FP, FP16, CAP_HWCAP, KERNEL_HWCAP_FPHP), |
2944 | HWCAP_CAP(ID_AA64PFR0_EL1, AdvSIMD, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMD), |
2945 | HWCAP_CAP(ID_AA64PFR0_EL1, AdvSIMD, FP16, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP), |
2946 | HWCAP_CAP(ID_AA64PFR0_EL1, DIT, IMP, CAP_HWCAP, KERNEL_HWCAP_DIT), |
2947 | HWCAP_CAP(ID_AA64PFR2_EL1, FPMR, IMP, CAP_HWCAP, KERNEL_HWCAP_FPMR), |
2948 | HWCAP_CAP(ID_AA64ISAR1_EL1, DPB, IMP, CAP_HWCAP, KERNEL_HWCAP_DCPOP), |
2949 | HWCAP_CAP(ID_AA64ISAR1_EL1, DPB, DPB2, CAP_HWCAP, KERNEL_HWCAP_DCPODP), |
2950 | HWCAP_CAP(ID_AA64ISAR1_EL1, JSCVT, IMP, CAP_HWCAP, KERNEL_HWCAP_JSCVT), |
2951 | HWCAP_CAP(ID_AA64ISAR1_EL1, FCMA, IMP, CAP_HWCAP, KERNEL_HWCAP_FCMA), |
2952 | HWCAP_CAP(ID_AA64ISAR1_EL1, LRCPC, IMP, CAP_HWCAP, KERNEL_HWCAP_LRCPC), |
2953 | HWCAP_CAP(ID_AA64ISAR1_EL1, LRCPC, LRCPC2, CAP_HWCAP, KERNEL_HWCAP_ILRCPC), |
2954 | HWCAP_CAP(ID_AA64ISAR1_EL1, LRCPC, LRCPC3, CAP_HWCAP, KERNEL_HWCAP_LRCPC3), |
2955 | HWCAP_CAP(ID_AA64ISAR1_EL1, FRINTTS, IMP, CAP_HWCAP, KERNEL_HWCAP_FRINT), |
2956 | HWCAP_CAP(ID_AA64ISAR1_EL1, SB, IMP, CAP_HWCAP, KERNEL_HWCAP_SB), |
2957 | HWCAP_CAP(ID_AA64ISAR1_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_BF16), |
2958 | HWCAP_CAP(ID_AA64ISAR1_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_EBF16), |
2959 | HWCAP_CAP(ID_AA64ISAR1_EL1, DGH, IMP, CAP_HWCAP, KERNEL_HWCAP_DGH), |
2960 | HWCAP_CAP(ID_AA64ISAR1_EL1, I8MM, IMP, CAP_HWCAP, KERNEL_HWCAP_I8MM), |
2961 | HWCAP_CAP(ID_AA64ISAR2_EL1, LUT, IMP, CAP_HWCAP, KERNEL_HWCAP_LUT), |
2962 | HWCAP_CAP(ID_AA64ISAR3_EL1, FAMINMAX, IMP, CAP_HWCAP, KERNEL_HWCAP_FAMINMAX), |
2963 | HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT), |
2964 | #ifdef CONFIG_ARM64_SVE |
2965 | HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), |
2966 | HWCAP_CAP(ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1), |
2967 | HWCAP_CAP(ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), |
2968 | HWCAP_CAP(ID_AA64ZFR0_EL1, AES, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES), |
2969 | HWCAP_CAP(ID_AA64ZFR0_EL1, AES, PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL), |
2970 | HWCAP_CAP(ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), |
2971 | HWCAP_CAP(ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16), |
2972 | HWCAP_CAP(ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), |
2973 | HWCAP_CAP(ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16), |
2974 | HWCAP_CAP(ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), |
2975 | HWCAP_CAP(ID_AA64ZFR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESM4), |
2976 | HWCAP_CAP(ID_AA64ZFR0_EL1, I8MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEI8MM), |
2977 | HWCAP_CAP(ID_AA64ZFR0_EL1, F32MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF32MM), |
2978 | HWCAP_CAP(ID_AA64ZFR0_EL1, F64MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM), |
2979 | #endif |
2980 | HWCAP_CAP(ID_AA64PFR1_EL1, SSBS, SSBS2, CAP_HWCAP, KERNEL_HWCAP_SSBS), |
2981 | #ifdef CONFIG_ARM64_BTI |
2982 | HWCAP_CAP(ID_AA64PFR1_EL1, BT, IMP, CAP_HWCAP, KERNEL_HWCAP_BTI), |
2983 | #endif |
2984 | #ifdef CONFIG_ARM64_PTR_AUTH |
2985 | HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, KERNEL_HWCAP_PACA), |
2986 | HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, KERNEL_HWCAP_PACG), |
2987 | #endif |
2988 | #ifdef CONFIG_ARM64_MTE |
2989 | HWCAP_CAP(ID_AA64PFR1_EL1, MTE, MTE2, CAP_HWCAP, KERNEL_HWCAP_MTE), |
2990 | HWCAP_CAP(ID_AA64PFR1_EL1, MTE, MTE3, CAP_HWCAP, KERNEL_HWCAP_MTE3), |
2991 | #endif /* CONFIG_ARM64_MTE */ |
2992 | HWCAP_CAP(ID_AA64MMFR0_EL1, ECV, IMP, CAP_HWCAP, KERNEL_HWCAP_ECV), |
2993 | HWCAP_CAP(ID_AA64MMFR1_EL1, AFP, IMP, CAP_HWCAP, KERNEL_HWCAP_AFP), |
2994 | HWCAP_CAP(ID_AA64ISAR2_EL1, CSSC, IMP, CAP_HWCAP, KERNEL_HWCAP_CSSC), |
2995 | HWCAP_CAP(ID_AA64ISAR2_EL1, RPRFM, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRFM), |
2996 | HWCAP_CAP(ID_AA64ISAR2_EL1, RPRES, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRES), |
2997 | HWCAP_CAP(ID_AA64ISAR2_EL1, WFxT, IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT), |
2998 | HWCAP_CAP(ID_AA64ISAR2_EL1, MOPS, IMP, CAP_HWCAP, KERNEL_HWCAP_MOPS), |
2999 | HWCAP_CAP(ID_AA64ISAR2_EL1, BC, IMP, CAP_HWCAP, KERNEL_HWCAP_HBC), |
3000 | #ifdef CONFIG_ARM64_SME |
3001 | HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), |
3002 | HWCAP_CAP(ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), |
3003 | HWCAP_CAP(ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2), |
3004 | HWCAP_CAP(ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1), |
3005 | HWCAP_CAP(ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2), |
3006 | HWCAP_CAP(ID_AA64SMFR0_EL1, I16I64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I64), |
3007 | HWCAP_CAP(ID_AA64SMFR0_EL1, F64F64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F64F64), |
3008 | HWCAP_CAP(ID_AA64SMFR0_EL1, I16I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I32), |
3009 | HWCAP_CAP(ID_AA64SMFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16B16), |
3010 | HWCAP_CAP(ID_AA64SMFR0_EL1, F16F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F16), |
3011 | HWCAP_CAP(ID_AA64SMFR0_EL1, F8F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F8F16), |
3012 | HWCAP_CAP(ID_AA64SMFR0_EL1, F8F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F8F32), |
3013 | HWCAP_CAP(ID_AA64SMFR0_EL1, I8I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I8I32), |
3014 | HWCAP_CAP(ID_AA64SMFR0_EL1, F16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F32), |
3015 | HWCAP_CAP(ID_AA64SMFR0_EL1, B16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16F32), |
3016 | HWCAP_CAP(ID_AA64SMFR0_EL1, BI32I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_BI32I32), |
3017 | HWCAP_CAP(ID_AA64SMFR0_EL1, F32F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F32F32), |
3018 | HWCAP_CAP(ID_AA64SMFR0_EL1, SF8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8FMA), |
3019 | HWCAP_CAP(ID_AA64SMFR0_EL1, SF8DP4, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP4), |
3020 | HWCAP_CAP(ID_AA64SMFR0_EL1, SF8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP2), |
3021 | #endif /* CONFIG_ARM64_SME */ |
3022 | HWCAP_CAP(ID_AA64FPFR0_EL1, F8CVT, IMP, CAP_HWCAP, KERNEL_HWCAP_F8CVT), |
3023 | HWCAP_CAP(ID_AA64FPFR0_EL1, F8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_F8FMA), |
3024 | HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP4), |
3025 | HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2), |
3026 | HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3), |
3027 | HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2), |
3028 | {}, |
3029 | }; |
3030 | |
3031 | #ifdef CONFIG_COMPAT |
3032 | static bool compat_has_neon(const struct arm64_cpu_capabilities *cap, int scope) |
3033 | { |
3034 | /* |
3035 | * Check that all of MVFR1_EL1.{SIMDSP, SIMDInt, SIMDLS} are available, |
3036 | * in line with that of arm32 as in vfp_init(). We make sure that the |
3037 | * check is future proof, by making sure value is non-zero. |
3038 | */ |
3039 | u32 mvfr1; |
3040 | |
3041 | WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible()); |
3042 | if (scope == SCOPE_SYSTEM) |
3043 | mvfr1 = read_sanitised_ftr_reg(SYS_MVFR1_EL1); |
3044 | else |
3045 | mvfr1 = read_sysreg_s(SYS_MVFR1_EL1); |
3046 | |
3047 | return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDSP_SHIFT) && |
3048 | cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDInt_SHIFT) && |
3049 | cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDLS_SHIFT); |
3050 | } |
3051 | #endif |
3052 | |
3053 | static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = { |
3054 | #ifdef CONFIG_COMPAT |
3055 | HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON), |
3056 | HWCAP_CAP(MVFR1_EL1, SIMDFMAC, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4), |
3057 | /* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */ |
3058 | HWCAP_CAP(MVFR0_EL1, FPDP, VFPv3, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP), |
3059 | HWCAP_CAP(MVFR0_EL1, FPDP, VFPv3, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3), |
3060 | HWCAP_CAP(MVFR1_EL1, FPHP, FP16, CAP_COMPAT_HWCAP, COMPAT_HWCAP_FPHP), |
3061 | HWCAP_CAP(MVFR1_EL1, SIMDHP, SIMDHP_FLOAT, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDHP), |
3062 | HWCAP_CAP(ID_ISAR5_EL1, AES, VMULL, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL), |
3063 | HWCAP_CAP(ID_ISAR5_EL1, AES, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES), |
3064 | HWCAP_CAP(ID_ISAR5_EL1, SHA1, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1), |
3065 | HWCAP_CAP(ID_ISAR5_EL1, SHA2, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2), |
3066 | HWCAP_CAP(ID_ISAR5_EL1, CRC32, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32), |
3067 | HWCAP_CAP(ID_ISAR6_EL1, DP, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDDP), |
3068 | HWCAP_CAP(ID_ISAR6_EL1, FHM, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDFHM), |
3069 | HWCAP_CAP(ID_ISAR6_EL1, SB, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SB), |
3070 | HWCAP_CAP(ID_ISAR6_EL1, BF16, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDBF16), |
3071 | HWCAP_CAP(ID_ISAR6_EL1, I8MM, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_I8MM), |
3072 | HWCAP_CAP(ID_PFR2_EL1, SSBS, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SSBS), |
3073 | #endif |
3074 | {}, |
3075 | }; |
3076 | |
3077 | static void cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap) |
3078 | { |
3079 | switch (cap->hwcap_type) { |
3080 | case CAP_HWCAP: |
3081 | cpu_set_feature(cap->hwcap); |
3082 | break; |
3083 | #ifdef CONFIG_COMPAT |
3084 | case CAP_COMPAT_HWCAP: |
3085 | compat_elf_hwcap |= (u32)cap->hwcap; |
3086 | break; |
3087 | case CAP_COMPAT_HWCAP2: |
3088 | compat_elf_hwcap2 |= (u32)cap->hwcap; |
3089 | break; |
3090 | #endif |
3091 | default: |
3092 | WARN_ON(1); |
3093 | break; |
3094 | } |
3095 | } |
3096 | |
3097 | /* Check if we have a particular HWCAP enabled */ |
3098 | static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap) |
3099 | { |
3100 | bool rc; |
3101 | |
3102 | switch (cap->hwcap_type) { |
3103 | case CAP_HWCAP: |
3104 | rc = cpu_have_feature(cap->hwcap); |
3105 | break; |
3106 | #ifdef CONFIG_COMPAT |
3107 | case CAP_COMPAT_HWCAP: |
3108 | rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0; |
3109 | break; |
3110 | case CAP_COMPAT_HWCAP2: |
3111 | rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0; |
3112 | break; |
3113 | #endif |
3114 | default: |
3115 | WARN_ON(1); |
3116 | rc = false; |
3117 | } |
3118 | |
3119 | return rc; |
3120 | } |
3121 | |
3122 | static void setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps) |
3123 | { |
3124 | /* We support emulation of accesses to CPU ID feature registers */ |
3125 | cpu_set_named_feature(CPUID); |
3126 | for (; hwcaps->matches; hwcaps++) |
3127 | if (hwcaps->matches(hwcaps, cpucap_default_scope(hwcaps))) |
3128 | cap_set_elf_hwcap(cap: hwcaps); |
3129 | } |
3130 | |
3131 | static void update_cpu_capabilities(u16 scope_mask) |
3132 | { |
3133 | int i; |
3134 | const struct arm64_cpu_capabilities *caps; |
3135 | |
3136 | scope_mask &= ARM64_CPUCAP_SCOPE_MASK; |
3137 | for (i = 0; i < ARM64_NCAPS; i++) { |
3138 | caps = cpucap_ptrs[i]; |
3139 | if (!caps || !(caps->type & scope_mask) || |
3140 | cpus_have_cap(caps->capability) || |
3141 | !caps->matches(caps, cpucap_default_scope(caps))) |
3142 | continue; |
3143 | |
3144 | if (caps->desc && !caps->cpus) |
3145 | pr_info("detected: %s\n" , caps->desc); |
3146 | |
3147 | __set_bit(caps->capability, system_cpucaps); |
3148 | |
3149 | if ((scope_mask & SCOPE_BOOT_CPU) && (caps->type & SCOPE_BOOT_CPU)) |
3150 | set_bit(caps->capability, boot_cpucaps); |
3151 | } |
3152 | } |
3153 | |
3154 | /* |
3155 | * Enable all the available capabilities on this CPU. The capabilities |
3156 | * with BOOT_CPU scope are handled separately and hence skipped here. |
3157 | */ |
3158 | static int cpu_enable_non_boot_scope_capabilities(void *__unused) |
3159 | { |
3160 | int i; |
3161 | u16 non_boot_scope = SCOPE_ALL & ~SCOPE_BOOT_CPU; |
3162 | |
3163 | for_each_available_cap(i) { |
3164 | const struct arm64_cpu_capabilities *cap = cpucap_ptrs[i]; |
3165 | |
3166 | if (WARN_ON(!cap)) |
3167 | continue; |
3168 | |
3169 | if (!(cap->type & non_boot_scope)) |
3170 | continue; |
3171 | |
3172 | if (cap->cpu_enable) |
3173 | cap->cpu_enable(cap); |
3174 | } |
3175 | return 0; |
3176 | } |
3177 | |
3178 | /* |
3179 | * Run through the enabled capabilities and enable() it on all active |
3180 | * CPUs |
3181 | */ |
3182 | static void __init enable_cpu_capabilities(u16 scope_mask) |
3183 | { |
3184 | int i; |
3185 | const struct arm64_cpu_capabilities *caps; |
3186 | bool boot_scope; |
3187 | |
3188 | scope_mask &= ARM64_CPUCAP_SCOPE_MASK; |
3189 | boot_scope = !!(scope_mask & SCOPE_BOOT_CPU); |
3190 | |
3191 | for (i = 0; i < ARM64_NCAPS; i++) { |
3192 | caps = cpucap_ptrs[i]; |
3193 | if (!caps || !(caps->type & scope_mask) || |
3194 | !cpus_have_cap(caps->capability)) |
3195 | continue; |
3196 | |
3197 | if (boot_scope && caps->cpu_enable) |
3198 | /* |
3199 | * Capabilities with SCOPE_BOOT_CPU scope are finalised |
3200 | * before any secondary CPU boots. Thus, each secondary |
3201 | * will enable the capability as appropriate via |
3202 | * check_local_cpu_capabilities(). The only exception is |
3203 | * the boot CPU, for which the capability must be |
3204 | * enabled here. This approach avoids costly |
3205 | * stop_machine() calls for this case. |
3206 | */ |
3207 | caps->cpu_enable(caps); |
3208 | } |
3209 | |
3210 | /* |
3211 | * For all non-boot scope capabilities, use stop_machine() |
3212 | * as it schedules the work allowing us to modify PSTATE, |
3213 | * instead of on_each_cpu() which uses an IPI, giving us a |
3214 | * PSTATE that disappears when we return. |
3215 | */ |
3216 | if (!boot_scope) |
3217 | stop_machine(fn: cpu_enable_non_boot_scope_capabilities, |
3218 | NULL, cpu_online_mask); |
3219 | } |
3220 | |
3221 | /* |
3222 | * Run through the list of capabilities to check for conflicts. |
3223 | * If the system has already detected a capability, take necessary |
3224 | * action on this CPU. |
3225 | */ |
3226 | static void verify_local_cpu_caps(u16 scope_mask) |
3227 | { |
3228 | int i; |
3229 | bool cpu_has_cap, system_has_cap; |
3230 | const struct arm64_cpu_capabilities *caps; |
3231 | |
3232 | scope_mask &= ARM64_CPUCAP_SCOPE_MASK; |
3233 | |
3234 | for (i = 0; i < ARM64_NCAPS; i++) { |
3235 | caps = cpucap_ptrs[i]; |
3236 | if (!caps || !(caps->type & scope_mask)) |
3237 | continue; |
3238 | |
3239 | cpu_has_cap = caps->matches(caps, SCOPE_LOCAL_CPU); |
3240 | system_has_cap = cpus_have_cap(caps->capability); |
3241 | |
3242 | if (system_has_cap) { |
3243 | /* |
3244 | * Check if the new CPU misses an advertised feature, |
3245 | * which is not safe to miss. |
3246 | */ |
3247 | if (!cpu_has_cap && !cpucap_late_cpu_optional(caps)) |
3248 | break; |
3249 | /* |
3250 | * We have to issue cpu_enable() irrespective of |
3251 | * whether the CPU has it or not, as it is enabeld |
3252 | * system wide. It is upto the call back to take |
3253 | * appropriate action on this CPU. |
3254 | */ |
3255 | if (caps->cpu_enable) |
3256 | caps->cpu_enable(caps); |
3257 | } else { |
3258 | /* |
3259 | * Check if the CPU has this capability if it isn't |
3260 | * safe to have when the system doesn't. |
3261 | */ |
3262 | if (cpu_has_cap && !cpucap_late_cpu_permitted(caps)) |
3263 | break; |
3264 | } |
3265 | } |
3266 | |
3267 | if (i < ARM64_NCAPS) { |
3268 | pr_crit("CPU%d: Detected conflict for capability %d (%s), System: %d, CPU: %d\n" , |
3269 | smp_processor_id(), caps->capability, |
3270 | caps->desc, system_has_cap, cpu_has_cap); |
3271 | |
3272 | if (cpucap_panic_on_conflict(cap: caps)) |
3273 | cpu_panic_kernel(); |
3274 | else |
3275 | cpu_die_early(); |
3276 | } |
3277 | } |
3278 | |
3279 | /* |
3280 | * Check for CPU features that are used in early boot |
3281 | * based on the Boot CPU value. |
3282 | */ |
3283 | static void check_early_cpu_features(void) |
3284 | { |
3285 | verify_cpu_asid_bits(); |
3286 | |
3287 | verify_local_cpu_caps(SCOPE_BOOT_CPU); |
3288 | } |
3289 | |
3290 | static void |
3291 | __verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps) |
3292 | { |
3293 | |
3294 | for (; caps->matches; caps++) |
3295 | if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) { |
3296 | pr_crit("CPU%d: missing HWCAP: %s\n" , |
3297 | smp_processor_id(), caps->desc); |
3298 | cpu_die_early(); |
3299 | } |
3300 | } |
3301 | |
3302 | static void verify_local_elf_hwcaps(void) |
3303 | { |
3304 | __verify_local_elf_hwcaps(caps: arm64_elf_hwcaps); |
3305 | |
3306 | if (id_aa64pfr0_32bit_el0(read_cpuid(ID_AA64PFR0_EL1))) |
3307 | __verify_local_elf_hwcaps(caps: compat_elf_hwcaps); |
3308 | } |
3309 | |
3310 | static void verify_sve_features(void) |
3311 | { |
3312 | unsigned long cpacr = cpacr_save_enable_kernel_sve(); |
3313 | |
3314 | if (vec_verify_vq_map(ARM64_VEC_SVE)) { |
3315 | pr_crit("CPU%d: SVE: vector length support mismatch\n" , |
3316 | smp_processor_id()); |
3317 | cpu_die_early(); |
3318 | } |
3319 | |
3320 | cpacr_restore(cpacr); |
3321 | } |
3322 | |
3323 | static void verify_sme_features(void) |
3324 | { |
3325 | unsigned long cpacr = cpacr_save_enable_kernel_sme(); |
3326 | |
3327 | if (vec_verify_vq_map(ARM64_VEC_SME)) { |
3328 | pr_crit("CPU%d: SME: vector length support mismatch\n" , |
3329 | smp_processor_id()); |
3330 | cpu_die_early(); |
3331 | } |
3332 | |
3333 | cpacr_restore(cpacr); |
3334 | } |
3335 | |
3336 | static void verify_hyp_capabilities(void) |
3337 | { |
3338 | u64 safe_mmfr1, mmfr0, mmfr1; |
3339 | int parange, ipa_max; |
3340 | unsigned int safe_vmid_bits, vmid_bits; |
3341 | |
3342 | if (!IS_ENABLED(CONFIG_KVM)) |
3343 | return; |
3344 | |
3345 | safe_mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); |
3346 | mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); |
3347 | mmfr1 = read_cpuid(ID_AA64MMFR1_EL1); |
3348 | |
3349 | /* Verify VMID bits */ |
3350 | safe_vmid_bits = get_vmid_bits(safe_mmfr1); |
3351 | vmid_bits = get_vmid_bits(mmfr1); |
3352 | if (vmid_bits < safe_vmid_bits) { |
3353 | pr_crit("CPU%d: VMID width mismatch\n" , smp_processor_id()); |
3354 | cpu_die_early(); |
3355 | } |
3356 | |
3357 | /* Verify IPA range */ |
3358 | parange = cpuid_feature_extract_unsigned_field(mmfr0, |
3359 | ID_AA64MMFR0_EL1_PARANGE_SHIFT); |
3360 | ipa_max = id_aa64mmfr0_parange_to_phys_shift(parange); |
3361 | if (ipa_max < get_kvm_ipa_limit()) { |
3362 | pr_crit("CPU%d: IPA range mismatch\n" , smp_processor_id()); |
3363 | cpu_die_early(); |
3364 | } |
3365 | } |
3366 | |
3367 | /* |
3368 | * Run through the enabled system capabilities and enable() it on this CPU. |
3369 | * The capabilities were decided based on the available CPUs at the boot time. |
3370 | * Any new CPU should match the system wide status of the capability. If the |
3371 | * new CPU doesn't have a capability which the system now has enabled, we |
3372 | * cannot do anything to fix it up and could cause unexpected failures. So |
3373 | * we park the CPU. |
3374 | */ |
3375 | static void verify_local_cpu_capabilities(void) |
3376 | { |
3377 | /* |
3378 | * The capabilities with SCOPE_BOOT_CPU are checked from |
3379 | * check_early_cpu_features(), as they need to be verified |
3380 | * on all secondary CPUs. |
3381 | */ |
3382 | verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU); |
3383 | verify_local_elf_hwcaps(); |
3384 | |
3385 | if (system_supports_sve()) |
3386 | verify_sve_features(); |
3387 | |
3388 | if (system_supports_sme()) |
3389 | verify_sme_features(); |
3390 | |
3391 | if (is_hyp_mode_available()) |
3392 | verify_hyp_capabilities(); |
3393 | } |
3394 | |
3395 | void check_local_cpu_capabilities(void) |
3396 | { |
3397 | /* |
3398 | * All secondary CPUs should conform to the early CPU features |
3399 | * in use by the kernel based on boot CPU. |
3400 | */ |
3401 | check_early_cpu_features(); |
3402 | |
3403 | /* |
3404 | * If we haven't finalised the system capabilities, this CPU gets |
3405 | * a chance to update the errata work arounds and local features. |
3406 | * Otherwise, this CPU should verify that it has all the system |
3407 | * advertised capabilities. |
3408 | */ |
3409 | if (!system_capabilities_finalized()) |
3410 | update_cpu_capabilities(SCOPE_LOCAL_CPU); |
3411 | else |
3412 | verify_local_cpu_capabilities(); |
3413 | } |
3414 | |
3415 | bool this_cpu_has_cap(unsigned int n) |
3416 | { |
3417 | if (!WARN_ON(preemptible()) && n < ARM64_NCAPS) { |
3418 | const struct arm64_cpu_capabilities *cap = cpucap_ptrs[n]; |
3419 | |
3420 | if (cap) |
3421 | return cap->matches(cap, SCOPE_LOCAL_CPU); |
3422 | } |
3423 | |
3424 | return false; |
3425 | } |
3426 | EXPORT_SYMBOL_GPL(this_cpu_has_cap); |
3427 | |
3428 | /* |
3429 | * This helper function is used in a narrow window when, |
3430 | * - The system wide safe registers are set with all the SMP CPUs and, |
3431 | * - The SYSTEM_FEATURE system_cpucaps may not have been set. |
3432 | */ |
3433 | static bool __maybe_unused __system_matches_cap(unsigned int n) |
3434 | { |
3435 | if (n < ARM64_NCAPS) { |
3436 | const struct arm64_cpu_capabilities *cap = cpucap_ptrs[n]; |
3437 | |
3438 | if (cap) |
3439 | return cap->matches(cap, SCOPE_SYSTEM); |
3440 | } |
3441 | return false; |
3442 | } |
3443 | |
3444 | void cpu_set_feature(unsigned int num) |
3445 | { |
3446 | set_bit(nr: num, addr: elf_hwcap); |
3447 | } |
3448 | |
3449 | bool cpu_have_feature(unsigned int num) |
3450 | { |
3451 | return test_bit(num, elf_hwcap); |
3452 | } |
3453 | EXPORT_SYMBOL_GPL(cpu_have_feature); |
3454 | |
3455 | unsigned long cpu_get_elf_hwcap(void) |
3456 | { |
3457 | /* |
3458 | * We currently only populate the first 32 bits of AT_HWCAP. Please |
3459 | * note that for userspace compatibility we guarantee that bits 62 |
3460 | * and 63 will always be returned as 0. |
3461 | */ |
3462 | return elf_hwcap[0]; |
3463 | } |
3464 | |
3465 | unsigned long cpu_get_elf_hwcap2(void) |
3466 | { |
3467 | return elf_hwcap[1]; |
3468 | } |
3469 | |
3470 | static void __init setup_boot_cpu_capabilities(void) |
3471 | { |
3472 | /* |
3473 | * The boot CPU's feature register values have been recorded. Detect |
3474 | * boot cpucaps and local cpucaps for the boot CPU, then enable and |
3475 | * patch alternatives for the available boot cpucaps. |
3476 | */ |
3477 | update_cpu_capabilities(SCOPE_BOOT_CPU | SCOPE_LOCAL_CPU); |
3478 | enable_cpu_capabilities(SCOPE_BOOT_CPU); |
3479 | apply_boot_alternatives(); |
3480 | } |
3481 | |
3482 | void __init setup_boot_cpu_features(void) |
3483 | { |
3484 | /* |
3485 | * Initialize the indirect array of CPU capabilities pointers before we |
3486 | * handle the boot CPU. |
3487 | */ |
3488 | init_cpucap_indirect_list(); |
3489 | |
3490 | /* |
3491 | * Detect broken pseudo-NMI. Must be called _before_ the call to |
3492 | * setup_boot_cpu_capabilities() since it interacts with |
3493 | * can_use_gic_priorities(). |
3494 | */ |
3495 | detect_system_supports_pseudo_nmi(); |
3496 | |
3497 | setup_boot_cpu_capabilities(); |
3498 | } |
3499 | |
3500 | static void __init setup_system_capabilities(void) |
3501 | { |
3502 | /* |
3503 | * The system-wide safe feature register values have been finalized. |
3504 | * Detect, enable, and patch alternatives for the available system |
3505 | * cpucaps. |
3506 | */ |
3507 | update_cpu_capabilities(SCOPE_SYSTEM); |
3508 | enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU); |
3509 | apply_alternatives_all(); |
3510 | |
3511 | /* |
3512 | * Log any cpucaps with a cpumask as these aren't logged by |
3513 | * update_cpu_capabilities(). |
3514 | */ |
3515 | for (int i = 0; i < ARM64_NCAPS; i++) { |
3516 | const struct arm64_cpu_capabilities *caps = cpucap_ptrs[i]; |
3517 | |
3518 | if (caps && caps->cpus && caps->desc && |
3519 | cpumask_any(caps->cpus) < nr_cpu_ids) |
3520 | pr_info("detected: %s on CPU%*pbl\n" , |
3521 | caps->desc, cpumask_pr_args(caps->cpus)); |
3522 | } |
3523 | |
3524 | /* |
3525 | * TTBR0 PAN doesn't have its own cpucap, so log it manually. |
3526 | */ |
3527 | if (system_uses_ttbr0_pan()) |
3528 | pr_info("emulated: Privileged Access Never (PAN) using TTBR0_EL1 switching\n" ); |
3529 | } |
3530 | |
3531 | void __init setup_system_features(void) |
3532 | { |
3533 | setup_system_capabilities(); |
3534 | |
3535 | kpti_install_ng_mappings(); |
3536 | |
3537 | sve_setup(); |
3538 | sme_setup(); |
3539 | |
3540 | /* |
3541 | * Check for sane CTR_EL0.CWG value. |
3542 | */ |
3543 | if (!cache_type_cwg()) |
3544 | pr_warn("No Cache Writeback Granule information, assuming %d\n" , |
3545 | ARCH_DMA_MINALIGN); |
3546 | } |
3547 | |
3548 | void __init setup_user_features(void) |
3549 | { |
3550 | user_feature_fixup(); |
3551 | |
3552 | setup_elf_hwcaps(arm64_elf_hwcaps); |
3553 | |
3554 | if (system_supports_32bit_el0()) { |
3555 | setup_elf_hwcaps(compat_elf_hwcaps); |
3556 | elf_hwcap_fixup(); |
3557 | } |
3558 | |
3559 | minsigstksz_setup(); |
3560 | } |
3561 | |
3562 | static int enable_mismatched_32bit_el0(unsigned int cpu) |
3563 | { |
3564 | /* |
3565 | * The first 32-bit-capable CPU we detected and so can no longer |
3566 | * be offlined by userspace. -1 indicates we haven't yet onlined |
3567 | * a 32-bit-capable CPU. |
3568 | */ |
3569 | static int lucky_winner = -1; |
3570 | |
3571 | struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu); |
3572 | bool cpu_32bit = id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0); |
3573 | |
3574 | if (cpu_32bit) { |
3575 | cpumask_set_cpu(cpu, dstp: cpu_32bit_el0_mask); |
3576 | static_branch_enable_cpuslocked(&arm64_mismatched_32bit_el0); |
3577 | } |
3578 | |
3579 | if (cpumask_test_cpu(cpu: 0, cpumask: cpu_32bit_el0_mask) == cpu_32bit) |
3580 | return 0; |
3581 | |
3582 | if (lucky_winner >= 0) |
3583 | return 0; |
3584 | |
3585 | /* |
3586 | * We've detected a mismatch. We need to keep one of our CPUs with |
3587 | * 32-bit EL0 online so that is_cpu_allowed() doesn't end up rejecting |
3588 | * every CPU in the system for a 32-bit task. |
3589 | */ |
3590 | lucky_winner = cpu_32bit ? cpu : cpumask_any_and(cpu_32bit_el0_mask, |
3591 | cpu_active_mask); |
3592 | get_cpu_device(cpu: lucky_winner)->offline_disabled = true; |
3593 | setup_elf_hwcaps(compat_elf_hwcaps); |
3594 | elf_hwcap_fixup(); |
3595 | pr_info("Asymmetric 32-bit EL0 support detected on CPU %u; CPU hot-unplug disabled on CPU %u\n" , |
3596 | cpu, lucky_winner); |
3597 | return 0; |
3598 | } |
3599 | |
3600 | static int __init init_32bit_el0_mask(void) |
3601 | { |
3602 | if (!allow_mismatched_32bit_el0) |
3603 | return 0; |
3604 | |
3605 | if (!zalloc_cpumask_var(mask: &cpu_32bit_el0_mask, GFP_KERNEL)) |
3606 | return -ENOMEM; |
3607 | |
3608 | return cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, |
3609 | name: "arm64/mismatched_32bit_el0:online" , |
3610 | startup: enable_mismatched_32bit_el0, NULL); |
3611 | } |
3612 | subsys_initcall_sync(init_32bit_el0_mask); |
3613 | |
3614 | static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap) |
3615 | { |
3616 | cpu_enable_swapper_cnp(); |
3617 | } |
3618 | |
3619 | /* |
3620 | * We emulate only the following system register space. |
3621 | * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 2 - 7] |
3622 | * See Table C5-6 System instruction encodings for System register accesses, |
3623 | * ARMv8 ARM(ARM DDI 0487A.f) for more details. |
3624 | */ |
3625 | static inline bool __attribute_const__ is_emulated(u32 id) |
3626 | { |
3627 | return (sys_reg_Op0(id) == 0x3 && |
3628 | sys_reg_CRn(id) == 0x0 && |
3629 | sys_reg_Op1(id) == 0x0 && |
3630 | (sys_reg_CRm(id) == 0 || |
3631 | ((sys_reg_CRm(id) >= 2) && (sys_reg_CRm(id) <= 7)))); |
3632 | } |
3633 | |
3634 | /* |
3635 | * With CRm == 0, reg should be one of : |
3636 | * MIDR_EL1, MPIDR_EL1 or REVIDR_EL1. |
3637 | */ |
3638 | static inline int emulate_id_reg(u32 id, u64 *valp) |
3639 | { |
3640 | switch (id) { |
3641 | case SYS_MIDR_EL1: |
3642 | *valp = read_cpuid_id(); |
3643 | break; |
3644 | case SYS_MPIDR_EL1: |
3645 | *valp = SYS_MPIDR_SAFE_VAL; |
3646 | break; |
3647 | case SYS_REVIDR_EL1: |
3648 | /* IMPLEMENTATION DEFINED values are emulated with 0 */ |
3649 | *valp = 0; |
3650 | break; |
3651 | default: |
3652 | return -EINVAL; |
3653 | } |
3654 | |
3655 | return 0; |
3656 | } |
3657 | |
3658 | static int emulate_sys_reg(u32 id, u64 *valp) |
3659 | { |
3660 | struct arm64_ftr_reg *regp; |
3661 | |
3662 | if (!is_emulated(id)) |
3663 | return -EINVAL; |
3664 | |
3665 | if (sys_reg_CRm(id) == 0) |
3666 | return emulate_id_reg(id, valp); |
3667 | |
3668 | regp = get_arm64_ftr_reg_nowarn(sys_id: id); |
3669 | if (regp) |
3670 | *valp = arm64_ftr_reg_user_value(regp); |
3671 | else |
3672 | /* |
3673 | * The untracked registers are either IMPLEMENTATION DEFINED |
3674 | * (e.g, ID_AFR0_EL1) or reserved RAZ. |
3675 | */ |
3676 | *valp = 0; |
3677 | return 0; |
3678 | } |
3679 | |
3680 | int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt) |
3681 | { |
3682 | int rc; |
3683 | u64 val; |
3684 | |
3685 | rc = emulate_sys_reg(id: sys_reg, valp: &val); |
3686 | if (!rc) { |
3687 | pt_regs_write_reg(regs, rt, val); |
3688 | arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); |
3689 | } |
3690 | return rc; |
3691 | } |
3692 | |
3693 | bool try_emulate_mrs(struct pt_regs *regs, u32 insn) |
3694 | { |
3695 | u32 sys_reg, rt; |
3696 | |
3697 | if (compat_user_mode(regs) || !aarch64_insn_is_mrs(insn)) |
3698 | return false; |
3699 | |
3700 | /* |
3701 | * sys_reg values are defined as used in mrs/msr instruction. |
3702 | * shift the imm value to get the encoding. |
3703 | */ |
3704 | sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5; |
3705 | rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn); |
3706 | return do_emulate_mrs(regs, sys_reg, rt) == 0; |
3707 | } |
3708 | |
3709 | enum mitigation_state arm64_get_meltdown_state(void) |
3710 | { |
3711 | if (__meltdown_safe) |
3712 | return SPECTRE_UNAFFECTED; |
3713 | |
3714 | if (arm64_kernel_unmapped_at_el0()) |
3715 | return SPECTRE_MITIGATED; |
3716 | |
3717 | return SPECTRE_VULNERABLE; |
3718 | } |
3719 | |
3720 | ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, |
3721 | char *buf) |
3722 | { |
3723 | switch (arm64_get_meltdown_state()) { |
3724 | case SPECTRE_UNAFFECTED: |
3725 | return sprintf(buf, fmt: "Not affected\n" ); |
3726 | |
3727 | case SPECTRE_MITIGATED: |
3728 | return sprintf(buf, fmt: "Mitigation: PTI\n" ); |
3729 | |
3730 | default: |
3731 | return sprintf(buf, fmt: "Vulnerable\n" ); |
3732 | } |
3733 | } |
3734 | |