1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Early cpufeature override framework |
4 | * |
5 | * Copyright (C) 2020 Google LLC |
6 | * Author: Marc Zyngier <maz@kernel.org> |
7 | */ |
8 | |
9 | #include <linux/ctype.h> |
10 | #include <linux/kernel.h> |
11 | #include <linux/libfdt.h> |
12 | |
13 | #include <asm/cacheflush.h> |
14 | #include <asm/cpufeature.h> |
15 | #include <asm/setup.h> |
16 | |
17 | #include "pi.h" |
18 | |
19 | #define FTR_DESC_NAME_LEN 20 |
20 | #define FTR_DESC_FIELD_LEN 10 |
21 | #define FTR_ALIAS_NAME_LEN 30 |
22 | #define FTR_ALIAS_OPTION_LEN 116 |
23 | |
24 | static u64 __boot_status __initdata; |
25 | |
26 | typedef bool filter_t(u64 val); |
27 | |
28 | struct ftr_set_desc { |
29 | char name[FTR_DESC_NAME_LEN]; |
30 | PREL64(struct arm64_ftr_override, override); |
31 | struct { |
32 | char name[FTR_DESC_FIELD_LEN]; |
33 | u8 shift; |
34 | u8 width; |
35 | PREL64(filter_t, filter); |
36 | } fields[]; |
37 | }; |
38 | |
39 | #define FIELD(n, s, f) { .name = n, .shift = s, .width = 4, .filter = f } |
40 | |
41 | static bool __init mmfr1_vh_filter(u64 val) |
42 | { |
43 | /* |
44 | * If we ever reach this point while running VHE, we're |
45 | * guaranteed to be on one of these funky, VHE-stuck CPUs. If |
46 | * the user was trying to force nVHE on us, proceed with |
47 | * attitude adjustment. |
48 | */ |
49 | return !(__boot_status == (BOOT_CPU_FLAG_E2H | BOOT_CPU_MODE_EL2) && |
50 | val == 0); |
51 | } |
52 | |
53 | static const struct ftr_set_desc mmfr1 __prel64_initconst = { |
54 | .name = "id_aa64mmfr1" , |
55 | .override = &id_aa64mmfr1_override, |
56 | .fields = { |
57 | FIELD("vh" , ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter), |
58 | {} |
59 | }, |
60 | }; |
61 | |
62 | |
63 | static bool __init mmfr2_varange_filter(u64 val) |
64 | { |
65 | int __maybe_unused feat; |
66 | |
67 | if (val) |
68 | return false; |
69 | |
70 | #ifdef CONFIG_ARM64_LPA2 |
71 | feat = cpuid_feature_extract_signed_field(read_sysreg(id_aa64mmfr0_el1), |
72 | ID_AA64MMFR0_EL1_TGRAN_SHIFT); |
73 | if (feat >= ID_AA64MMFR0_EL1_TGRAN_LPA2) { |
74 | id_aa64mmfr0_override.val |= |
75 | (ID_AA64MMFR0_EL1_TGRAN_LPA2 - 1) << ID_AA64MMFR0_EL1_TGRAN_SHIFT; |
76 | id_aa64mmfr0_override.mask |= 0xfU << ID_AA64MMFR0_EL1_TGRAN_SHIFT; |
77 | } |
78 | #endif |
79 | return true; |
80 | } |
81 | |
82 | static const struct ftr_set_desc mmfr2 __prel64_initconst = { |
83 | .name = "id_aa64mmfr2" , |
84 | .override = &id_aa64mmfr2_override, |
85 | .fields = { |
86 | FIELD("varange" , ID_AA64MMFR2_EL1_VARange_SHIFT, mmfr2_varange_filter), |
87 | {} |
88 | }, |
89 | }; |
90 | |
91 | static bool __init pfr0_sve_filter(u64 val) |
92 | { |
93 | /* |
94 | * Disabling SVE also means disabling all the features that |
95 | * are associated with it. The easiest way to do it is just to |
96 | * override id_aa64zfr0_el1 to be 0. |
97 | */ |
98 | if (!val) { |
99 | id_aa64zfr0_override.val = 0; |
100 | id_aa64zfr0_override.mask = GENMASK(63, 0); |
101 | } |
102 | |
103 | return true; |
104 | } |
105 | |
106 | static const struct ftr_set_desc pfr0 __prel64_initconst = { |
107 | .name = "id_aa64pfr0" , |
108 | .override = &id_aa64pfr0_override, |
109 | .fields = { |
110 | FIELD("sve" , ID_AA64PFR0_EL1_SVE_SHIFT, pfr0_sve_filter), |
111 | {} |
112 | }, |
113 | }; |
114 | |
115 | static bool __init pfr1_sme_filter(u64 val) |
116 | { |
117 | /* |
118 | * Similarly to SVE, disabling SME also means disabling all |
119 | * the features that are associated with it. Just set |
120 | * id_aa64smfr0_el1 to 0 and don't look back. |
121 | */ |
122 | if (!val) { |
123 | id_aa64smfr0_override.val = 0; |
124 | id_aa64smfr0_override.mask = GENMASK(63, 0); |
125 | } |
126 | |
127 | return true; |
128 | } |
129 | |
130 | static const struct ftr_set_desc pfr1 __prel64_initconst = { |
131 | .name = "id_aa64pfr1" , |
132 | .override = &id_aa64pfr1_override, |
133 | .fields = { |
134 | FIELD("bt" , ID_AA64PFR1_EL1_BT_SHIFT, NULL ), |
135 | FIELD("mte" , ID_AA64PFR1_EL1_MTE_SHIFT, NULL), |
136 | FIELD("sme" , ID_AA64PFR1_EL1_SME_SHIFT, pfr1_sme_filter), |
137 | {} |
138 | }, |
139 | }; |
140 | |
141 | static const struct ftr_set_desc isar1 __prel64_initconst = { |
142 | .name = "id_aa64isar1" , |
143 | .override = &id_aa64isar1_override, |
144 | .fields = { |
145 | FIELD("gpi" , ID_AA64ISAR1_EL1_GPI_SHIFT, NULL), |
146 | FIELD("gpa" , ID_AA64ISAR1_EL1_GPA_SHIFT, NULL), |
147 | FIELD("api" , ID_AA64ISAR1_EL1_API_SHIFT, NULL), |
148 | FIELD("apa" , ID_AA64ISAR1_EL1_APA_SHIFT, NULL), |
149 | {} |
150 | }, |
151 | }; |
152 | |
153 | static const struct ftr_set_desc isar2 __prel64_initconst = { |
154 | .name = "id_aa64isar2" , |
155 | .override = &id_aa64isar2_override, |
156 | .fields = { |
157 | FIELD("gpa3" , ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL), |
158 | FIELD("apa3" , ID_AA64ISAR2_EL1_APA3_SHIFT, NULL), |
159 | FIELD("mops" , ID_AA64ISAR2_EL1_MOPS_SHIFT, NULL), |
160 | {} |
161 | }, |
162 | }; |
163 | |
164 | static const struct ftr_set_desc smfr0 __prel64_initconst = { |
165 | .name = "id_aa64smfr0" , |
166 | .override = &id_aa64smfr0_override, |
167 | .fields = { |
168 | FIELD("smever" , ID_AA64SMFR0_EL1_SMEver_SHIFT, NULL), |
169 | /* FA64 is a one bit field... :-/ */ |
170 | { "fa64" , ID_AA64SMFR0_EL1_FA64_SHIFT, 1, }, |
171 | {} |
172 | }, |
173 | }; |
174 | |
175 | static bool __init hvhe_filter(u64 val) |
176 | { |
177 | u64 mmfr1 = read_sysreg(id_aa64mmfr1_el1); |
178 | |
179 | return (val == 1 && |
180 | lower_32_bits(__boot_status) == BOOT_CPU_MODE_EL2 && |
181 | cpuid_feature_extract_unsigned_field(mmfr1, |
182 | ID_AA64MMFR1_EL1_VH_SHIFT)); |
183 | } |
184 | |
185 | static const struct ftr_set_desc sw_features __prel64_initconst = { |
186 | .name = "arm64_sw" , |
187 | .override = &arm64_sw_feature_override, |
188 | .fields = { |
189 | FIELD("nokaslr" , ARM64_SW_FEATURE_OVERRIDE_NOKASLR, NULL), |
190 | FIELD("hvhe" , ARM64_SW_FEATURE_OVERRIDE_HVHE, hvhe_filter), |
191 | FIELD("rodataoff" , ARM64_SW_FEATURE_OVERRIDE_RODATA_OFF, NULL), |
192 | {} |
193 | }, |
194 | }; |
195 | |
196 | static const |
197 | PREL64(const struct ftr_set_desc, reg) regs[] __prel64_initconst = { |
198 | { .reg: &mmfr1 }, |
199 | { &mmfr2 }, |
200 | { &pfr0 }, |
201 | { &pfr1 }, |
202 | { &isar1 }, |
203 | { &isar2 }, |
204 | { &smfr0 }, |
205 | { &sw_features }, |
206 | }; |
207 | |
208 | static const struct { |
209 | char alias[FTR_ALIAS_NAME_LEN]; |
210 | char feature[FTR_ALIAS_OPTION_LEN]; |
211 | } aliases[] __initconst = { |
212 | { "kvm_arm.mode=nvhe" , "id_aa64mmfr1.vh=0" }, |
213 | { "kvm_arm.mode=protected" , "id_aa64mmfr1.vh=0" }, |
214 | { "arm64.nosve" , "id_aa64pfr0.sve=0" }, |
215 | { "arm64.nosme" , "id_aa64pfr1.sme=0" }, |
216 | { "arm64.nobti" , "id_aa64pfr1.bt=0" }, |
217 | { "arm64.nopauth" , |
218 | "id_aa64isar1.gpi=0 id_aa64isar1.gpa=0 " |
219 | "id_aa64isar1.api=0 id_aa64isar1.apa=0 " |
220 | "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0" }, |
221 | { "arm64.nomops" , "id_aa64isar2.mops=0" }, |
222 | { "arm64.nomte" , "id_aa64pfr1.mte=0" }, |
223 | { "nokaslr" , "arm64_sw.nokaslr=1" }, |
224 | { "rodata=off" , "arm64_sw.rodataoff=1" }, |
225 | { "arm64.nolva" , "id_aa64mmfr2.varange=0" }, |
226 | }; |
227 | |
228 | static int __init parse_hexdigit(const char *p, u64 *v) |
229 | { |
230 | // skip "0x" if it comes next |
231 | if (p[0] == '0' && tolower(p[1]) == 'x') |
232 | p += 2; |
233 | |
234 | // check whether the RHS is a single hex digit |
235 | if (!isxdigit(p[0]) || (p[1] && !isspace(p[1]))) |
236 | return -EINVAL; |
237 | |
238 | *v = tolower(*p) - (isdigit(c: *p) ? '0' : 'a' - 10); |
239 | return 0; |
240 | } |
241 | |
242 | static int __init find_field(const char *cmdline, char *opt, int len, |
243 | const struct ftr_set_desc *reg, int f, u64 *v) |
244 | { |
245 | int flen = strlen(reg->fields[f].name); |
246 | |
247 | // append '<fieldname>=' to obtain '<name>.<fieldname>=' |
248 | memcpy(opt + len, reg->fields[f].name, flen); |
249 | len += flen; |
250 | opt[len++] = '='; |
251 | |
252 | if (memcmp(p: cmdline, q: opt, size: len)) |
253 | return -1; |
254 | |
255 | return parse_hexdigit(p: cmdline + len, v); |
256 | } |
257 | |
258 | static void __init match_options(const char *cmdline) |
259 | { |
260 | char opt[FTR_DESC_NAME_LEN + FTR_DESC_FIELD_LEN + 2]; |
261 | int i; |
262 | |
263 | for (i = 0; i < ARRAY_SIZE(regs); i++) { |
264 | const struct ftr_set_desc *reg = prel64_pointer(regs[i].reg); |
265 | struct arm64_ftr_override *override; |
266 | int len = strlen(reg->name); |
267 | int f; |
268 | |
269 | override = prel64_pointer(reg->override); |
270 | |
271 | // set opt[] to '<name>.' |
272 | memcpy(opt, reg->name, len); |
273 | opt[len++] = '.'; |
274 | |
275 | for (f = 0; reg->fields[f].name[0] != '\0'; f++) { |
276 | u64 shift = reg->fields[f].shift; |
277 | u64 width = reg->fields[f].width ?: 4; |
278 | u64 mask = GENMASK_ULL(shift + width - 1, shift); |
279 | bool (*filter)(u64 val); |
280 | u64 v; |
281 | |
282 | if (find_field(cmdline, opt, len, reg, f, v: &v)) |
283 | continue; |
284 | |
285 | /* |
286 | * If an override gets filtered out, advertise |
287 | * it by setting the value to the all-ones while |
288 | * clearing the mask... Yes, this is fragile. |
289 | */ |
290 | filter = prel64_pointer(reg->fields[f].filter); |
291 | if (filter && !filter(v)) { |
292 | override->val |= mask; |
293 | override->mask &= ~mask; |
294 | continue; |
295 | } |
296 | |
297 | override->val &= ~mask; |
298 | override->val |= (v << shift) & mask; |
299 | override->mask |= mask; |
300 | |
301 | return; |
302 | } |
303 | } |
304 | } |
305 | |
306 | static __init void __parse_cmdline(const char *cmdline, bool parse_aliases) |
307 | { |
308 | do { |
309 | char buf[256]; |
310 | size_t len; |
311 | int i; |
312 | |
313 | cmdline = skip_spaces(cmdline); |
314 | |
315 | /* terminate on "--" appearing on the command line by itself */ |
316 | if (cmdline[0] == '-' && cmdline[1] == '-' && isspace(cmdline[2])) |
317 | return; |
318 | |
319 | for (len = 0; cmdline[len] && !isspace(cmdline[len]); len++) { |
320 | if (len >= sizeof(buf) - 1) |
321 | break; |
322 | if (cmdline[len] == '-') |
323 | buf[len] = '_'; |
324 | else |
325 | buf[len] = cmdline[len]; |
326 | } |
327 | if (!len) |
328 | return; |
329 | |
330 | buf[len] = 0; |
331 | |
332 | cmdline += len; |
333 | |
334 | match_options(cmdline: buf); |
335 | |
336 | for (i = 0; parse_aliases && i < ARRAY_SIZE(aliases); i++) |
337 | if (!memcmp(p: buf, q: aliases[i].alias, size: len + 1)) |
338 | __parse_cmdline(cmdline: aliases[i].feature, parse_aliases: false); |
339 | } while (1); |
340 | } |
341 | |
342 | static __init const u8 *get_bootargs_cmdline(const void *fdt, int node) |
343 | { |
344 | static char const bootargs[] __initconst = "bootargs" ; |
345 | const u8 *prop; |
346 | |
347 | if (node < 0) |
348 | return NULL; |
349 | |
350 | prop = fdt_getprop(fdt, nodeoffset: node, name: bootargs, NULL); |
351 | if (!prop) |
352 | return NULL; |
353 | |
354 | return strlen(prop) ? prop : NULL; |
355 | } |
356 | |
357 | static __init void parse_cmdline(const void *fdt, int chosen) |
358 | { |
359 | static char const cmdline[] __initconst = CONFIG_CMDLINE; |
360 | const u8 *prop = get_bootargs_cmdline(fdt, node: chosen); |
361 | |
362 | if (IS_ENABLED(CONFIG_CMDLINE_FORCE) || !prop) |
363 | __parse_cmdline(cmdline, parse_aliases: true); |
364 | |
365 | if (!IS_ENABLED(CONFIG_CMDLINE_FORCE) && prop) |
366 | __parse_cmdline(cmdline: prop, parse_aliases: true); |
367 | } |
368 | |
369 | void __init init_feature_override(u64 boot_status, const void *fdt, |
370 | int chosen) |
371 | { |
372 | struct arm64_ftr_override *override; |
373 | const struct ftr_set_desc *reg; |
374 | int i; |
375 | |
376 | for (i = 0; i < ARRAY_SIZE(regs); i++) { |
377 | reg = prel64_pointer(regs[i].reg); |
378 | override = prel64_pointer(reg->override); |
379 | |
380 | override->val = 0; |
381 | override->mask = 0; |
382 | } |
383 | |
384 | __boot_status = boot_status; |
385 | |
386 | parse_cmdline(fdt, chosen); |
387 | |
388 | for (i = 0; i < ARRAY_SIZE(regs); i++) { |
389 | reg = prel64_pointer(regs[i].reg); |
390 | override = prel64_pointer(reg->override); |
391 | dcache_clean_inval_poc((unsigned long)override, |
392 | (unsigned long)(override + 1)); |
393 | } |
394 | } |
395 | |
396 | char * __init skip_spaces(const char *str) |
397 | { |
398 | while (isspace(*str)) |
399 | ++str; |
400 | return (char *)str; |
401 | } |
402 | |