1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Handle detection, reporting and mitigation of Spectre v1, v2, v3a and v4, as |
4 | * detailed at: |
5 | * |
6 | * https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability |
7 | * |
8 | * This code was originally written hastily under an awful lot of stress and so |
9 | * aspects of it are somewhat hacky. Unfortunately, changing anything in here |
10 | * instantly makes me feel ill. Thanks, Jann. Thann. |
11 | * |
12 | * Copyright (C) 2018 ARM Ltd, All Rights Reserved. |
13 | * Copyright (C) 2020 Google LLC |
14 | * |
15 | * "If there's something strange in your neighbourhood, who you gonna call?" |
16 | * |
17 | * Authors: Will Deacon <will@kernel.org> and Marc Zyngier <maz@kernel.org> |
18 | */ |
19 | |
20 | #include <linux/arm-smccc.h> |
21 | #include <linux/bpf.h> |
22 | #include <linux/cpu.h> |
23 | #include <linux/device.h> |
24 | #include <linux/nospec.h> |
25 | #include <linux/prctl.h> |
26 | #include <linux/sched/task_stack.h> |
27 | |
28 | #include <asm/debug-monitors.h> |
29 | #include <asm/insn.h> |
30 | #include <asm/spectre.h> |
31 | #include <asm/traps.h> |
32 | #include <asm/vectors.h> |
33 | #include <asm/virt.h> |
34 | |
35 | /* |
36 | * We try to ensure that the mitigation state can never change as the result of |
37 | * onlining a late CPU. |
38 | */ |
39 | static void update_mitigation_state(enum mitigation_state *oldp, |
40 | enum mitigation_state new) |
41 | { |
42 | enum mitigation_state state; |
43 | |
44 | do { |
45 | state = READ_ONCE(*oldp); |
46 | if (new <= state) |
47 | break; |
48 | |
49 | /* Userspace almost certainly can't deal with this. */ |
50 | if (WARN_ON(system_capabilities_finalized())) |
51 | break; |
52 | } while (cmpxchg_relaxed(oldp, state, new) != state); |
53 | } |
54 | |
55 | /* |
56 | * Spectre v1. |
57 | * |
58 | * The kernel can't protect userspace for this one: it's each person for |
59 | * themselves. Advertise what we're doing and be done with it. |
60 | */ |
61 | ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, |
62 | char *buf) |
63 | { |
64 | return sprintf(buf, fmt: "Mitigation: __user pointer sanitization\n" ); |
65 | } |
66 | |
67 | /* |
68 | * Spectre v2. |
69 | * |
70 | * This one sucks. A CPU is either: |
71 | * |
72 | * - Mitigated in hardware and advertised by ID_AA64PFR0_EL1.CSV2. |
73 | * - Mitigated in hardware and listed in our "safe list". |
74 | * - Mitigated in software by firmware. |
75 | * - Mitigated in software by a CPU-specific dance in the kernel and a |
76 | * firmware call at EL2. |
77 | * - Vulnerable. |
78 | * |
79 | * It's not unlikely for different CPUs in a big.LITTLE system to fall into |
80 | * different camps. |
81 | */ |
82 | static enum mitigation_state spectre_v2_state; |
83 | |
84 | static bool __read_mostly __nospectre_v2; |
85 | static int __init parse_spectre_v2_param(char *str) |
86 | { |
87 | __nospectre_v2 = true; |
88 | return 0; |
89 | } |
90 | early_param("nospectre_v2" , parse_spectre_v2_param); |
91 | |
92 | static bool spectre_v2_mitigations_off(void) |
93 | { |
94 | bool ret = __nospectre_v2 || cpu_mitigations_off(); |
95 | |
96 | if (ret) |
97 | pr_info_once("spectre-v2 mitigation disabled by command line option\n" ); |
98 | |
99 | return ret; |
100 | } |
101 | |
102 | static const char *get_bhb_affected_string(enum mitigation_state bhb_state) |
103 | { |
104 | switch (bhb_state) { |
105 | case SPECTRE_UNAFFECTED: |
106 | return "" ; |
107 | default: |
108 | case SPECTRE_VULNERABLE: |
109 | return ", but not BHB" ; |
110 | case SPECTRE_MITIGATED: |
111 | return ", BHB" ; |
112 | } |
113 | } |
114 | |
115 | static bool _unprivileged_ebpf_enabled(void) |
116 | { |
117 | #ifdef CONFIG_BPF_SYSCALL |
118 | return !sysctl_unprivileged_bpf_disabled; |
119 | #else |
120 | return false; |
121 | #endif |
122 | } |
123 | |
124 | ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, |
125 | char *buf) |
126 | { |
127 | enum mitigation_state bhb_state = arm64_get_spectre_bhb_state(); |
128 | const char *bhb_str = get_bhb_affected_string(bhb_state: bhb_state); |
129 | const char *v2_str = "Branch predictor hardening" ; |
130 | |
131 | switch (spectre_v2_state) { |
132 | case SPECTRE_UNAFFECTED: |
133 | if (bhb_state == SPECTRE_UNAFFECTED) |
134 | return sprintf(buf, fmt: "Not affected\n" ); |
135 | |
136 | /* |
137 | * Platforms affected by Spectre-BHB can't report |
138 | * "Not affected" for Spectre-v2. |
139 | */ |
140 | v2_str = "CSV2" ; |
141 | fallthrough; |
142 | case SPECTRE_MITIGATED: |
143 | if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled()) |
144 | return sprintf(buf, fmt: "Vulnerable: Unprivileged eBPF enabled\n" ); |
145 | |
146 | return sprintf(buf, fmt: "Mitigation: %s%s\n" , v2_str, bhb_str); |
147 | case SPECTRE_VULNERABLE: |
148 | fallthrough; |
149 | default: |
150 | return sprintf(buf, fmt: "Vulnerable\n" ); |
151 | } |
152 | } |
153 | |
154 | static enum mitigation_state spectre_v2_get_cpu_hw_mitigation_state(void) |
155 | { |
156 | u64 pfr0; |
157 | static const struct midr_range spectre_v2_safe_list[] = { |
158 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A35), |
159 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A53), |
160 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A55), |
161 | MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53), |
162 | MIDR_ALL_VERSIONS(MIDR_HISI_TSV110), |
163 | MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER), |
164 | MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER), |
165 | MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER), |
166 | { /* sentinel */ } |
167 | }; |
168 | |
169 | /* If the CPU has CSV2 set, we're safe */ |
170 | pfr0 = read_cpuid(ID_AA64PFR0_EL1); |
171 | if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_CSV2_SHIFT)) |
172 | return SPECTRE_UNAFFECTED; |
173 | |
174 | /* Alternatively, we have a list of unaffected CPUs */ |
175 | if (is_midr_in_range_list(read_cpuid_id(), spectre_v2_safe_list)) |
176 | return SPECTRE_UNAFFECTED; |
177 | |
178 | return SPECTRE_VULNERABLE; |
179 | } |
180 | |
181 | static enum mitigation_state spectre_v2_get_cpu_fw_mitigation_state(void) |
182 | { |
183 | int ret; |
184 | struct arm_smccc_res res; |
185 | |
186 | arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
187 | ARM_SMCCC_ARCH_WORKAROUND_1, &res); |
188 | |
189 | ret = res.a0; |
190 | switch (ret) { |
191 | case SMCCC_RET_SUCCESS: |
192 | return SPECTRE_MITIGATED; |
193 | case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: |
194 | return SPECTRE_UNAFFECTED; |
195 | default: |
196 | fallthrough; |
197 | case SMCCC_RET_NOT_SUPPORTED: |
198 | return SPECTRE_VULNERABLE; |
199 | } |
200 | } |
201 | |
202 | bool has_spectre_v2(const struct arm64_cpu_capabilities *entry, int scope) |
203 | { |
204 | WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); |
205 | |
206 | if (spectre_v2_get_cpu_hw_mitigation_state() == SPECTRE_UNAFFECTED) |
207 | return false; |
208 | |
209 | if (spectre_v2_get_cpu_fw_mitigation_state() == SPECTRE_UNAFFECTED) |
210 | return false; |
211 | |
212 | return true; |
213 | } |
214 | |
215 | enum mitigation_state arm64_get_spectre_v2_state(void) |
216 | { |
217 | return spectre_v2_state; |
218 | } |
219 | |
220 | DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); |
221 | |
222 | static void install_bp_hardening_cb(bp_hardening_cb_t fn) |
223 | { |
224 | __this_cpu_write(bp_hardening_data.fn, fn); |
225 | |
226 | /* |
227 | * Vinz Clortho takes the hyp_vecs start/end "keys" at |
228 | * the door when we're a guest. Skip the hyp-vectors work. |
229 | */ |
230 | if (!is_hyp_mode_available()) |
231 | return; |
232 | |
233 | __this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT); |
234 | } |
235 | |
236 | /* Called during entry so must be noinstr */ |
237 | static noinstr void call_smc_arch_workaround_1(void) |
238 | { |
239 | arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); |
240 | } |
241 | |
242 | /* Called during entry so must be noinstr */ |
243 | static noinstr void call_hvc_arch_workaround_1(void) |
244 | { |
245 | arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); |
246 | } |
247 | |
248 | /* Called during entry so must be noinstr */ |
249 | static noinstr void qcom_link_stack_sanitisation(void) |
250 | { |
251 | u64 tmp; |
252 | |
253 | asm volatile("mov %0, x30 \n" |
254 | ".rept 16 \n" |
255 | "bl . + 4 \n" |
256 | ".endr \n" |
257 | "mov x30, %0 \n" |
258 | : "=&r" (tmp)); |
259 | } |
260 | |
261 | static bp_hardening_cb_t spectre_v2_get_sw_mitigation_cb(void) |
262 | { |
263 | u32 midr = read_cpuid_id(); |
264 | if (((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR) && |
265 | ((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR_V1)) |
266 | return NULL; |
267 | |
268 | return qcom_link_stack_sanitisation; |
269 | } |
270 | |
271 | static enum mitigation_state spectre_v2_enable_fw_mitigation(void) |
272 | { |
273 | bp_hardening_cb_t cb; |
274 | enum mitigation_state state; |
275 | |
276 | state = spectre_v2_get_cpu_fw_mitigation_state(); |
277 | if (state != SPECTRE_MITIGATED) |
278 | return state; |
279 | |
280 | if (spectre_v2_mitigations_off()) |
281 | return SPECTRE_VULNERABLE; |
282 | |
283 | switch (arm_smccc_1_1_get_conduit()) { |
284 | case SMCCC_CONDUIT_HVC: |
285 | cb = call_hvc_arch_workaround_1; |
286 | break; |
287 | |
288 | case SMCCC_CONDUIT_SMC: |
289 | cb = call_smc_arch_workaround_1; |
290 | break; |
291 | |
292 | default: |
293 | return SPECTRE_VULNERABLE; |
294 | } |
295 | |
296 | /* |
297 | * Prefer a CPU-specific workaround if it exists. Note that we |
298 | * still rely on firmware for the mitigation at EL2. |
299 | */ |
300 | cb = spectre_v2_get_sw_mitigation_cb() ?: cb; |
301 | install_bp_hardening_cb(cb); |
302 | return SPECTRE_MITIGATED; |
303 | } |
304 | |
305 | void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused) |
306 | { |
307 | enum mitigation_state state; |
308 | |
309 | WARN_ON(preemptible()); |
310 | |
311 | state = spectre_v2_get_cpu_hw_mitigation_state(); |
312 | if (state == SPECTRE_VULNERABLE) |
313 | state = spectre_v2_enable_fw_mitigation(); |
314 | |
315 | update_mitigation_state(oldp: &spectre_v2_state, new: state); |
316 | } |
317 | |
318 | /* |
319 | * Spectre-v3a. |
320 | * |
321 | * Phew, there's not an awful lot to do here! We just instruct EL2 to use |
322 | * an indirect trampoline for the hyp vectors so that guests can't read |
323 | * VBAR_EL2 to defeat randomisation of the hypervisor VA layout. |
324 | */ |
325 | bool has_spectre_v3a(const struct arm64_cpu_capabilities *entry, int scope) |
326 | { |
327 | static const struct midr_range spectre_v3a_unsafe_list[] = { |
328 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), |
329 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), |
330 | {}, |
331 | }; |
332 | |
333 | WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); |
334 | return is_midr_in_range_list(read_cpuid_id(), spectre_v3a_unsafe_list); |
335 | } |
336 | |
337 | void spectre_v3a_enable_mitigation(const struct arm64_cpu_capabilities *__unused) |
338 | { |
339 | struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); |
340 | |
341 | if (this_cpu_has_cap(ARM64_SPECTRE_V3A)) |
342 | data->slot += HYP_VECTOR_INDIRECT; |
343 | } |
344 | |
345 | /* |
346 | * Spectre v4. |
347 | * |
348 | * If you thought Spectre v2 was nasty, wait until you see this mess. A CPU is |
349 | * either: |
350 | * |
351 | * - Mitigated in hardware and listed in our "safe list". |
352 | * - Mitigated in hardware via PSTATE.SSBS. |
353 | * - Mitigated in software by firmware (sometimes referred to as SSBD). |
354 | * |
355 | * Wait, that doesn't sound so bad, does it? Keep reading... |
356 | * |
357 | * A major source of headaches is that the software mitigation is enabled both |
358 | * on a per-task basis, but can also be forced on for the kernel, necessitating |
359 | * both context-switch *and* entry/exit hooks. To make it even worse, some CPUs |
360 | * allow EL0 to toggle SSBS directly, which can end up with the prctl() state |
361 | * being stale when re-entering the kernel. The usual big.LITTLE caveats apply, |
362 | * so you can have systems that have both firmware and SSBS mitigations. This |
363 | * means we actually have to reject late onlining of CPUs with mitigations if |
364 | * all of the currently onlined CPUs are safelisted, as the mitigation tends to |
365 | * be opt-in for userspace. Yes, really, the cure is worse than the disease. |
366 | * |
367 | * The only good part is that if the firmware mitigation is present, then it is |
368 | * present for all CPUs, meaning we don't have to worry about late onlining of a |
369 | * vulnerable CPU if one of the boot CPUs is using the firmware mitigation. |
370 | * |
371 | * Give me a VAX-11/780 any day of the week... |
372 | */ |
373 | static enum mitigation_state spectre_v4_state; |
374 | |
375 | /* This is the per-cpu state tracking whether we need to talk to firmware */ |
376 | DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required); |
377 | |
378 | enum spectre_v4_policy { |
379 | SPECTRE_V4_POLICY_MITIGATION_DYNAMIC, |
380 | SPECTRE_V4_POLICY_MITIGATION_ENABLED, |
381 | SPECTRE_V4_POLICY_MITIGATION_DISABLED, |
382 | }; |
383 | |
384 | static enum spectre_v4_policy __read_mostly __spectre_v4_policy; |
385 | |
386 | static const struct spectre_v4_param { |
387 | const char *str; |
388 | enum spectre_v4_policy policy; |
389 | } spectre_v4_params[] = { |
390 | { "force-on" , SPECTRE_V4_POLICY_MITIGATION_ENABLED, }, |
391 | { "force-off" , SPECTRE_V4_POLICY_MITIGATION_DISABLED, }, |
392 | { "kernel" , SPECTRE_V4_POLICY_MITIGATION_DYNAMIC, }, |
393 | }; |
394 | static int __init parse_spectre_v4_param(char *str) |
395 | { |
396 | int i; |
397 | |
398 | if (!str || !str[0]) |
399 | return -EINVAL; |
400 | |
401 | for (i = 0; i < ARRAY_SIZE(spectre_v4_params); i++) { |
402 | const struct spectre_v4_param *param = &spectre_v4_params[i]; |
403 | |
404 | if (strncmp(str, param->str, strlen(param->str))) |
405 | continue; |
406 | |
407 | __spectre_v4_policy = param->policy; |
408 | return 0; |
409 | } |
410 | |
411 | return -EINVAL; |
412 | } |
413 | early_param("ssbd" , parse_spectre_v4_param); |
414 | |
415 | /* |
416 | * Because this was all written in a rush by people working in different silos, |
417 | * we've ended up with multiple command line options to control the same thing. |
418 | * Wrap these up in some helpers, which prefer disabling the mitigation if faced |
419 | * with contradictory parameters. The mitigation is always either "off", |
420 | * "dynamic" or "on". |
421 | */ |
422 | static bool spectre_v4_mitigations_off(void) |
423 | { |
424 | bool ret = cpu_mitigations_off() || |
425 | __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DISABLED; |
426 | |
427 | if (ret) |
428 | pr_info_once("spectre-v4 mitigation disabled by command-line option\n" ); |
429 | |
430 | return ret; |
431 | } |
432 | |
433 | /* Do we need to toggle the mitigation state on entry to/exit from the kernel? */ |
434 | static bool spectre_v4_mitigations_dynamic(void) |
435 | { |
436 | return !spectre_v4_mitigations_off() && |
437 | __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DYNAMIC; |
438 | } |
439 | |
440 | static bool spectre_v4_mitigations_on(void) |
441 | { |
442 | return !spectre_v4_mitigations_off() && |
443 | __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_ENABLED; |
444 | } |
445 | |
446 | ssize_t cpu_show_spec_store_bypass(struct device *dev, |
447 | struct device_attribute *attr, char *buf) |
448 | { |
449 | switch (spectre_v4_state) { |
450 | case SPECTRE_UNAFFECTED: |
451 | return sprintf(buf, fmt: "Not affected\n" ); |
452 | case SPECTRE_MITIGATED: |
453 | return sprintf(buf, fmt: "Mitigation: Speculative Store Bypass disabled via prctl\n" ); |
454 | case SPECTRE_VULNERABLE: |
455 | fallthrough; |
456 | default: |
457 | return sprintf(buf, fmt: "Vulnerable\n" ); |
458 | } |
459 | } |
460 | |
461 | enum mitigation_state arm64_get_spectre_v4_state(void) |
462 | { |
463 | return spectre_v4_state; |
464 | } |
465 | |
466 | static enum mitigation_state spectre_v4_get_cpu_hw_mitigation_state(void) |
467 | { |
468 | static const struct midr_range spectre_v4_safe_list[] = { |
469 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A35), |
470 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A53), |
471 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A55), |
472 | MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53), |
473 | MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER), |
474 | MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER), |
475 | { /* sentinel */ }, |
476 | }; |
477 | |
478 | if (is_midr_in_range_list(read_cpuid_id(), spectre_v4_safe_list)) |
479 | return SPECTRE_UNAFFECTED; |
480 | |
481 | /* CPU features are detected first */ |
482 | if (this_cpu_has_cap(ARM64_SSBS)) |
483 | return SPECTRE_MITIGATED; |
484 | |
485 | return SPECTRE_VULNERABLE; |
486 | } |
487 | |
488 | static enum mitigation_state spectre_v4_get_cpu_fw_mitigation_state(void) |
489 | { |
490 | int ret; |
491 | struct arm_smccc_res res; |
492 | |
493 | arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
494 | ARM_SMCCC_ARCH_WORKAROUND_2, &res); |
495 | |
496 | ret = res.a0; |
497 | switch (ret) { |
498 | case SMCCC_RET_SUCCESS: |
499 | return SPECTRE_MITIGATED; |
500 | case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: |
501 | fallthrough; |
502 | case SMCCC_RET_NOT_REQUIRED: |
503 | return SPECTRE_UNAFFECTED; |
504 | default: |
505 | fallthrough; |
506 | case SMCCC_RET_NOT_SUPPORTED: |
507 | return SPECTRE_VULNERABLE; |
508 | } |
509 | } |
510 | |
511 | bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope) |
512 | { |
513 | enum mitigation_state state; |
514 | |
515 | WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); |
516 | |
517 | state = spectre_v4_get_cpu_hw_mitigation_state(); |
518 | if (state == SPECTRE_VULNERABLE) |
519 | state = spectre_v4_get_cpu_fw_mitigation_state(); |
520 | |
521 | return state != SPECTRE_UNAFFECTED; |
522 | } |
523 | |
524 | bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr) |
525 | { |
526 | const u32 instr_mask = ~(1U << PSTATE_Imm_shift); |
527 | const u32 instr_val = 0xd500401f | PSTATE_SSBS; |
528 | |
529 | if ((instr & instr_mask) != instr_val) |
530 | return false; |
531 | |
532 | if (instr & BIT(PSTATE_Imm_shift)) |
533 | regs->pstate |= PSR_SSBS_BIT; |
534 | else |
535 | regs->pstate &= ~PSR_SSBS_BIT; |
536 | |
537 | arm64_skip_faulting_instruction(regs, 4); |
538 | return true; |
539 | } |
540 | |
541 | static enum mitigation_state spectre_v4_enable_hw_mitigation(void) |
542 | { |
543 | enum mitigation_state state; |
544 | |
545 | /* |
546 | * If the system is mitigated but this CPU doesn't have SSBS, then |
547 | * we must be on the safelist and there's nothing more to do. |
548 | */ |
549 | state = spectre_v4_get_cpu_hw_mitigation_state(); |
550 | if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS)) |
551 | return state; |
552 | |
553 | if (spectre_v4_mitigations_off()) { |
554 | sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS); |
555 | set_pstate_ssbs(1); |
556 | return SPECTRE_VULNERABLE; |
557 | } |
558 | |
559 | /* SCTLR_EL1.DSSBS was initialised to 0 during boot */ |
560 | set_pstate_ssbs(0); |
561 | return SPECTRE_MITIGATED; |
562 | } |
563 | |
564 | /* |
565 | * Patch a branch over the Spectre-v4 mitigation code with a NOP so that |
566 | * we fallthrough and check whether firmware needs to be called on this CPU. |
567 | */ |
568 | void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt, |
569 | __le32 *origptr, |
570 | __le32 *updptr, int nr_inst) |
571 | { |
572 | BUG_ON(nr_inst != 1); /* Branch -> NOP */ |
573 | |
574 | if (spectre_v4_mitigations_off()) |
575 | return; |
576 | |
577 | if (cpus_have_cap(ARM64_SSBS)) |
578 | return; |
579 | |
580 | if (spectre_v4_mitigations_dynamic()) |
581 | *updptr = cpu_to_le32(aarch64_insn_gen_nop()); |
582 | } |
583 | |
584 | /* |
585 | * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction |
586 | * to call into firmware to adjust the mitigation state. |
587 | */ |
588 | void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt, |
589 | __le32 *origptr, |
590 | __le32 *updptr, int nr_inst) |
591 | { |
592 | u32 insn; |
593 | |
594 | BUG_ON(nr_inst != 1); /* NOP -> HVC/SMC */ |
595 | |
596 | switch (arm_smccc_1_1_get_conduit()) { |
597 | case SMCCC_CONDUIT_HVC: |
598 | insn = aarch64_insn_get_hvc_value(); |
599 | break; |
600 | case SMCCC_CONDUIT_SMC: |
601 | insn = aarch64_insn_get_smc_value(); |
602 | break; |
603 | default: |
604 | return; |
605 | } |
606 | |
607 | *updptr = cpu_to_le32(insn); |
608 | } |
609 | |
610 | static enum mitigation_state spectre_v4_enable_fw_mitigation(void) |
611 | { |
612 | enum mitigation_state state; |
613 | |
614 | state = spectre_v4_get_cpu_fw_mitigation_state(); |
615 | if (state != SPECTRE_MITIGATED) |
616 | return state; |
617 | |
618 | if (spectre_v4_mitigations_off()) { |
619 | arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, false, NULL); |
620 | return SPECTRE_VULNERABLE; |
621 | } |
622 | |
623 | arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, true, NULL); |
624 | |
625 | if (spectre_v4_mitigations_dynamic()) |
626 | __this_cpu_write(arm64_ssbd_callback_required, 1); |
627 | |
628 | return SPECTRE_MITIGATED; |
629 | } |
630 | |
631 | void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused) |
632 | { |
633 | enum mitigation_state state; |
634 | |
635 | WARN_ON(preemptible()); |
636 | |
637 | state = spectre_v4_enable_hw_mitigation(); |
638 | if (state == SPECTRE_VULNERABLE) |
639 | state = spectre_v4_enable_fw_mitigation(); |
640 | |
641 | update_mitigation_state(oldp: &spectre_v4_state, new: state); |
642 | } |
643 | |
644 | static void __update_pstate_ssbs(struct pt_regs *regs, bool state) |
645 | { |
646 | u64 bit = compat_user_mode(regs) ? PSR_AA32_SSBS_BIT : PSR_SSBS_BIT; |
647 | |
648 | if (state) |
649 | regs->pstate |= bit; |
650 | else |
651 | regs->pstate &= ~bit; |
652 | } |
653 | |
654 | void spectre_v4_enable_task_mitigation(struct task_struct *tsk) |
655 | { |
656 | struct pt_regs *regs = task_pt_regs(tsk); |
657 | bool ssbs = false, kthread = tsk->flags & PF_KTHREAD; |
658 | |
659 | if (spectre_v4_mitigations_off()) |
660 | ssbs = true; |
661 | else if (spectre_v4_mitigations_dynamic() && !kthread) |
662 | ssbs = !test_tsk_thread_flag(tsk, TIF_SSBD); |
663 | |
664 | __update_pstate_ssbs(regs, state: ssbs); |
665 | } |
666 | |
667 | /* |
668 | * The Spectre-v4 mitigation can be controlled via a prctl() from userspace. |
669 | * This is interesting because the "speculation disabled" behaviour can be |
670 | * configured so that it is preserved across exec(), which means that the |
671 | * prctl() may be necessary even when PSTATE.SSBS can be toggled directly |
672 | * from userspace. |
673 | */ |
674 | static void ssbd_prctl_enable_mitigation(struct task_struct *task) |
675 | { |
676 | task_clear_spec_ssb_noexec(p: task); |
677 | task_set_spec_ssb_disable(p: task); |
678 | set_tsk_thread_flag(tsk: task, TIF_SSBD); |
679 | } |
680 | |
681 | static void ssbd_prctl_disable_mitigation(struct task_struct *task) |
682 | { |
683 | task_clear_spec_ssb_noexec(p: task); |
684 | task_clear_spec_ssb_disable(p: task); |
685 | clear_tsk_thread_flag(tsk: task, TIF_SSBD); |
686 | } |
687 | |
688 | static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl) |
689 | { |
690 | switch (ctrl) { |
691 | case PR_SPEC_ENABLE: |
692 | /* Enable speculation: disable mitigation */ |
693 | /* |
694 | * Force disabled speculation prevents it from being |
695 | * re-enabled. |
696 | */ |
697 | if (task_spec_ssb_force_disable(p: task)) |
698 | return -EPERM; |
699 | |
700 | /* |
701 | * If the mitigation is forced on, then speculation is forced |
702 | * off and we again prevent it from being re-enabled. |
703 | */ |
704 | if (spectre_v4_mitigations_on()) |
705 | return -EPERM; |
706 | |
707 | ssbd_prctl_disable_mitigation(task); |
708 | break; |
709 | case PR_SPEC_FORCE_DISABLE: |
710 | /* Force disable speculation: force enable mitigation */ |
711 | /* |
712 | * If the mitigation is forced off, then speculation is forced |
713 | * on and we prevent it from being disabled. |
714 | */ |
715 | if (spectre_v4_mitigations_off()) |
716 | return -EPERM; |
717 | |
718 | task_set_spec_ssb_force_disable(p: task); |
719 | fallthrough; |
720 | case PR_SPEC_DISABLE: |
721 | /* Disable speculation: enable mitigation */ |
722 | /* Same as PR_SPEC_FORCE_DISABLE */ |
723 | if (spectre_v4_mitigations_off()) |
724 | return -EPERM; |
725 | |
726 | ssbd_prctl_enable_mitigation(task); |
727 | break; |
728 | case PR_SPEC_DISABLE_NOEXEC: |
729 | /* Disable speculation until execve(): enable mitigation */ |
730 | /* |
731 | * If the mitigation state is forced one way or the other, then |
732 | * we must fail now before we try to toggle it on execve(). |
733 | */ |
734 | if (task_spec_ssb_force_disable(p: task) || |
735 | spectre_v4_mitigations_off() || |
736 | spectre_v4_mitigations_on()) { |
737 | return -EPERM; |
738 | } |
739 | |
740 | ssbd_prctl_enable_mitigation(task); |
741 | task_set_spec_ssb_noexec(p: task); |
742 | break; |
743 | default: |
744 | return -ERANGE; |
745 | } |
746 | |
747 | spectre_v4_enable_task_mitigation(tsk: task); |
748 | return 0; |
749 | } |
750 | |
751 | int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, |
752 | unsigned long ctrl) |
753 | { |
754 | switch (which) { |
755 | case PR_SPEC_STORE_BYPASS: |
756 | return ssbd_prctl_set(task, ctrl); |
757 | default: |
758 | return -ENODEV; |
759 | } |
760 | } |
761 | |
762 | static int ssbd_prctl_get(struct task_struct *task) |
763 | { |
764 | switch (spectre_v4_state) { |
765 | case SPECTRE_UNAFFECTED: |
766 | return PR_SPEC_NOT_AFFECTED; |
767 | case SPECTRE_MITIGATED: |
768 | if (spectre_v4_mitigations_on()) |
769 | return PR_SPEC_NOT_AFFECTED; |
770 | |
771 | if (spectre_v4_mitigations_dynamic()) |
772 | break; |
773 | |
774 | /* Mitigations are disabled, so we're vulnerable. */ |
775 | fallthrough; |
776 | case SPECTRE_VULNERABLE: |
777 | fallthrough; |
778 | default: |
779 | return PR_SPEC_ENABLE; |
780 | } |
781 | |
782 | /* Check the mitigation state for this task */ |
783 | if (task_spec_ssb_force_disable(p: task)) |
784 | return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; |
785 | |
786 | if (task_spec_ssb_noexec(p: task)) |
787 | return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; |
788 | |
789 | if (task_spec_ssb_disable(p: task)) |
790 | return PR_SPEC_PRCTL | PR_SPEC_DISABLE; |
791 | |
792 | return PR_SPEC_PRCTL | PR_SPEC_ENABLE; |
793 | } |
794 | |
795 | int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) |
796 | { |
797 | switch (which) { |
798 | case PR_SPEC_STORE_BYPASS: |
799 | return ssbd_prctl_get(task); |
800 | default: |
801 | return -ENODEV; |
802 | } |
803 | } |
804 | |
805 | /* |
806 | * Spectre BHB. |
807 | * |
808 | * A CPU is either: |
809 | * - Mitigated by a branchy loop a CPU specific number of times, and listed |
810 | * in our "loop mitigated list". |
811 | * - Mitigated in software by the firmware Spectre v2 call. |
812 | * - Has the ClearBHB instruction to perform the mitigation. |
813 | * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no |
814 | * software mitigation in the vectors is needed. |
815 | * - Has CSV2.3, so is unaffected. |
816 | */ |
817 | static enum mitigation_state spectre_bhb_state; |
818 | |
819 | enum mitigation_state arm64_get_spectre_bhb_state(void) |
820 | { |
821 | return spectre_bhb_state; |
822 | } |
823 | |
824 | enum bhb_mitigation_bits { |
825 | BHB_LOOP, |
826 | BHB_FW, |
827 | BHB_HW, |
828 | BHB_INSN, |
829 | }; |
830 | static unsigned long system_bhb_mitigations; |
831 | |
832 | /* |
833 | * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any |
834 | * SCOPE_SYSTEM call will give the right answer. |
835 | */ |
836 | u8 spectre_bhb_loop_affected(int scope) |
837 | { |
838 | u8 k = 0; |
839 | static u8 max_bhb_k; |
840 | |
841 | if (scope == SCOPE_LOCAL_CPU) { |
842 | static const struct midr_range spectre_bhb_k32_list[] = { |
843 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), |
844 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE), |
845 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), |
846 | MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), |
847 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), |
848 | MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), |
849 | MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), |
850 | MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), |
851 | {}, |
852 | }; |
853 | static const struct midr_range spectre_bhb_k24_list[] = { |
854 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), |
855 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), |
856 | MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), |
857 | {}, |
858 | }; |
859 | static const struct midr_range spectre_bhb_k11_list[] = { |
860 | MIDR_ALL_VERSIONS(MIDR_AMPERE1), |
861 | {}, |
862 | }; |
863 | static const struct midr_range spectre_bhb_k8_list[] = { |
864 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), |
865 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), |
866 | {}, |
867 | }; |
868 | |
869 | if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list)) |
870 | k = 32; |
871 | else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list)) |
872 | k = 24; |
873 | else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k11_list)) |
874 | k = 11; |
875 | else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list)) |
876 | k = 8; |
877 | |
878 | max_bhb_k = max(max_bhb_k, k); |
879 | } else { |
880 | k = max_bhb_k; |
881 | } |
882 | |
883 | return k; |
884 | } |
885 | |
886 | static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void) |
887 | { |
888 | int ret; |
889 | struct arm_smccc_res res; |
890 | |
891 | arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
892 | ARM_SMCCC_ARCH_WORKAROUND_3, &res); |
893 | |
894 | ret = res.a0; |
895 | switch (ret) { |
896 | case SMCCC_RET_SUCCESS: |
897 | return SPECTRE_MITIGATED; |
898 | case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: |
899 | return SPECTRE_UNAFFECTED; |
900 | default: |
901 | fallthrough; |
902 | case SMCCC_RET_NOT_SUPPORTED: |
903 | return SPECTRE_VULNERABLE; |
904 | } |
905 | } |
906 | |
907 | static bool is_spectre_bhb_fw_affected(int scope) |
908 | { |
909 | static bool system_affected; |
910 | enum mitigation_state fw_state; |
911 | bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE; |
912 | static const struct midr_range spectre_bhb_firmware_mitigated_list[] = { |
913 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), |
914 | MIDR_ALL_VERSIONS(MIDR_CORTEX_A75), |
915 | {}, |
916 | }; |
917 | bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(), |
918 | spectre_bhb_firmware_mitigated_list); |
919 | |
920 | if (scope != SCOPE_LOCAL_CPU) |
921 | return system_affected; |
922 | |
923 | fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); |
924 | if (cpu_in_list || (has_smccc && fw_state == SPECTRE_MITIGATED)) { |
925 | system_affected = true; |
926 | return true; |
927 | } |
928 | |
929 | return false; |
930 | } |
931 | |
932 | static bool supports_ecbhb(int scope) |
933 | { |
934 | u64 mmfr1; |
935 | |
936 | if (scope == SCOPE_LOCAL_CPU) |
937 | mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1); |
938 | else |
939 | mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); |
940 | |
941 | return cpuid_feature_extract_unsigned_field(mmfr1, |
942 | ID_AA64MMFR1_EL1_ECBHB_SHIFT); |
943 | } |
944 | |
945 | bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, |
946 | int scope) |
947 | { |
948 | WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); |
949 | |
950 | if (supports_csv2p3(scope)) |
951 | return false; |
952 | |
953 | if (supports_clearbhb(scope)) |
954 | return true; |
955 | |
956 | if (spectre_bhb_loop_affected(scope)) |
957 | return true; |
958 | |
959 | if (is_spectre_bhb_fw_affected(scope)) |
960 | return true; |
961 | |
962 | return false; |
963 | } |
964 | |
965 | static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) |
966 | { |
967 | const char *v = arm64_get_bp_hardening_vector(slot); |
968 | |
969 | __this_cpu_write(this_cpu_vector, v); |
970 | |
971 | /* |
972 | * When KPTI is in use, the vectors are switched when exiting to |
973 | * user-space. |
974 | */ |
975 | if (cpus_have_cap(ARM64_UNMAP_KERNEL_AT_EL0)) |
976 | return; |
977 | |
978 | write_sysreg(v, vbar_el1); |
979 | isb(); |
980 | } |
981 | |
982 | static bool __read_mostly __nospectre_bhb; |
983 | static int __init parse_spectre_bhb_param(char *str) |
984 | { |
985 | __nospectre_bhb = true; |
986 | return 0; |
987 | } |
988 | early_param("nospectre_bhb" , parse_spectre_bhb_param); |
989 | |
990 | void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) |
991 | { |
992 | bp_hardening_cb_t cpu_cb; |
993 | enum mitigation_state fw_state, state = SPECTRE_VULNERABLE; |
994 | struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); |
995 | |
996 | if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU)) |
997 | return; |
998 | |
999 | if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) { |
1000 | /* No point mitigating Spectre-BHB alone. */ |
1001 | } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { |
1002 | pr_info_once("spectre-bhb mitigation disabled by compile time option\n" ); |
1003 | } else if (cpu_mitigations_off() || __nospectre_bhb) { |
1004 | pr_info_once("spectre-bhb mitigation disabled by command line option\n" ); |
1005 | } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { |
1006 | state = SPECTRE_MITIGATED; |
1007 | set_bit(nr: BHB_HW, addr: &system_bhb_mitigations); |
1008 | } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) { |
1009 | /* |
1010 | * Ensure KVM uses the indirect vector which will have ClearBHB |
1011 | * added. |
1012 | */ |
1013 | if (!data->slot) |
1014 | data->slot = HYP_VECTOR_INDIRECT; |
1015 | |
1016 | this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN); |
1017 | state = SPECTRE_MITIGATED; |
1018 | set_bit(nr: BHB_INSN, addr: &system_bhb_mitigations); |
1019 | } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) { |
1020 | /* |
1021 | * Ensure KVM uses the indirect vector which will have the |
1022 | * branchy-loop added. A57/A72-r0 will already have selected |
1023 | * the spectre-indirect vector, which is sufficient for BHB |
1024 | * too. |
1025 | */ |
1026 | if (!data->slot) |
1027 | data->slot = HYP_VECTOR_INDIRECT; |
1028 | |
1029 | this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP); |
1030 | state = SPECTRE_MITIGATED; |
1031 | set_bit(nr: BHB_LOOP, addr: &system_bhb_mitigations); |
1032 | } else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) { |
1033 | fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); |
1034 | if (fw_state == SPECTRE_MITIGATED) { |
1035 | /* |
1036 | * Ensure KVM uses one of the spectre bp_hardening |
1037 | * vectors. The indirect vector doesn't include the EL3 |
1038 | * call, so needs upgrading to |
1039 | * HYP_VECTOR_SPECTRE_INDIRECT. |
1040 | */ |
1041 | if (!data->slot || data->slot == HYP_VECTOR_INDIRECT) |
1042 | data->slot += 1; |
1043 | |
1044 | this_cpu_set_vectors(EL1_VECTOR_BHB_FW); |
1045 | |
1046 | /* |
1047 | * The WA3 call in the vectors supersedes the WA1 call |
1048 | * made during context-switch. Uninstall any firmware |
1049 | * bp_hardening callback. |
1050 | */ |
1051 | cpu_cb = spectre_v2_get_sw_mitigation_cb(); |
1052 | if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb) |
1053 | __this_cpu_write(bp_hardening_data.fn, NULL); |
1054 | |
1055 | state = SPECTRE_MITIGATED; |
1056 | set_bit(nr: BHB_FW, addr: &system_bhb_mitigations); |
1057 | } |
1058 | } |
1059 | |
1060 | update_mitigation_state(oldp: &spectre_bhb_state, new: state); |
1061 | } |
1062 | |
1063 | /* Patched to NOP when enabled */ |
1064 | void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt, |
1065 | __le32 *origptr, |
1066 | __le32 *updptr, int nr_inst) |
1067 | { |
1068 | BUG_ON(nr_inst != 1); |
1069 | |
1070 | if (test_bit(BHB_LOOP, &system_bhb_mitigations)) |
1071 | *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); |
1072 | } |
1073 | |
1074 | /* Patched to NOP when enabled */ |
1075 | void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt, |
1076 | __le32 *origptr, |
1077 | __le32 *updptr, int nr_inst) |
1078 | { |
1079 | BUG_ON(nr_inst != 1); |
1080 | |
1081 | if (test_bit(BHB_FW, &system_bhb_mitigations)) |
1082 | *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); |
1083 | } |
1084 | |
1085 | /* Patched to correct the immediate */ |
1086 | void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt, |
1087 | __le32 *origptr, __le32 *updptr, int nr_inst) |
1088 | { |
1089 | u8 rd; |
1090 | u32 insn; |
1091 | u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM); |
1092 | |
1093 | BUG_ON(nr_inst != 1); /* MOV -> MOV */ |
1094 | |
1095 | if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) |
1096 | return; |
1097 | |
1098 | insn = le32_to_cpu(*origptr); |
1099 | rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn); |
1100 | insn = aarch64_insn_gen_movewide(rd, loop_count, 0, |
1101 | AARCH64_INSN_VARIANT_64BIT, |
1102 | AARCH64_INSN_MOVEWIDE_ZERO); |
1103 | *updptr++ = cpu_to_le32(insn); |
1104 | } |
1105 | |
1106 | /* Patched to mov WA3 when supported */ |
1107 | void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt, |
1108 | __le32 *origptr, __le32 *updptr, int nr_inst) |
1109 | { |
1110 | u8 rd; |
1111 | u32 insn; |
1112 | |
1113 | BUG_ON(nr_inst != 1); /* MOV -> MOV */ |
1114 | |
1115 | if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) || |
1116 | !test_bit(BHB_FW, &system_bhb_mitigations)) |
1117 | return; |
1118 | |
1119 | insn = le32_to_cpu(*origptr); |
1120 | rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn); |
1121 | |
1122 | insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR, |
1123 | AARCH64_INSN_VARIANT_32BIT, |
1124 | AARCH64_INSN_REG_ZR, rd, |
1125 | ARM_SMCCC_ARCH_WORKAROUND_3); |
1126 | if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT)) |
1127 | return; |
1128 | |
1129 | *updptr++ = cpu_to_le32(insn); |
1130 | } |
1131 | |
1132 | /* Patched to NOP when not supported */ |
1133 | void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt, |
1134 | __le32 *origptr, __le32 *updptr, int nr_inst) |
1135 | { |
1136 | BUG_ON(nr_inst != 2); |
1137 | |
1138 | if (test_bit(BHB_INSN, &system_bhb_mitigations)) |
1139 | return; |
1140 | |
1141 | *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); |
1142 | *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); |
1143 | } |
1144 | |
1145 | #ifdef CONFIG_BPF_SYSCALL |
1146 | #define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n" |
1147 | void unpriv_ebpf_notify(int new_state) |
1148 | { |
1149 | if (spectre_v2_state == SPECTRE_VULNERABLE || |
1150 | spectre_bhb_state != SPECTRE_MITIGATED) |
1151 | return; |
1152 | |
1153 | if (!new_state) |
1154 | pr_err("WARNING: %s" , EBPF_WARN); |
1155 | } |
1156 | #endif |
1157 | |