1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Handle detection, reporting and mitigation of Spectre v1, v2, v3a and v4, as
4 * detailed at:
5 *
6 * https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability
7 *
8 * This code was originally written hastily under an awful lot of stress and so
9 * aspects of it are somewhat hacky. Unfortunately, changing anything in here
10 * instantly makes me feel ill. Thanks, Jann. Thann.
11 *
12 * Copyright (C) 2018 ARM Ltd, All Rights Reserved.
13 * Copyright (C) 2020 Google LLC
14 *
15 * "If there's something strange in your neighbourhood, who you gonna call?"
16 *
17 * Authors: Will Deacon <will@kernel.org> and Marc Zyngier <maz@kernel.org>
18 */
19
20#include <linux/arm-smccc.h>
21#include <linux/bpf.h>
22#include <linux/cpu.h>
23#include <linux/device.h>
24#include <linux/nospec.h>
25#include <linux/prctl.h>
26#include <linux/sched/task_stack.h>
27
28#include <asm/debug-monitors.h>
29#include <asm/insn.h>
30#include <asm/spectre.h>
31#include <asm/traps.h>
32#include <asm/vectors.h>
33#include <asm/virt.h>
34
35/*
36 * We try to ensure that the mitigation state can never change as the result of
37 * onlining a late CPU.
38 */
39static void update_mitigation_state(enum mitigation_state *oldp,
40 enum mitigation_state new)
41{
42 enum mitigation_state state;
43
44 do {
45 state = READ_ONCE(*oldp);
46 if (new <= state)
47 break;
48
49 /* Userspace almost certainly can't deal with this. */
50 if (WARN_ON(system_capabilities_finalized()))
51 break;
52 } while (cmpxchg_relaxed(oldp, state, new) != state);
53}
54
55/*
56 * Spectre v1.
57 *
58 * The kernel can't protect userspace for this one: it's each person for
59 * themselves. Advertise what we're doing and be done with it.
60 */
61ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
62 char *buf)
63{
64 return sprintf(buf, fmt: "Mitigation: __user pointer sanitization\n");
65}
66
67/*
68 * Spectre v2.
69 *
70 * This one sucks. A CPU is either:
71 *
72 * - Mitigated in hardware and advertised by ID_AA64PFR0_EL1.CSV2.
73 * - Mitigated in hardware and listed in our "safe list".
74 * - Mitigated in software by firmware.
75 * - Mitigated in software by a CPU-specific dance in the kernel and a
76 * firmware call at EL2.
77 * - Vulnerable.
78 *
79 * It's not unlikely for different CPUs in a big.LITTLE system to fall into
80 * different camps.
81 */
82static enum mitigation_state spectre_v2_state;
83
84static bool __read_mostly __nospectre_v2;
85static int __init parse_spectre_v2_param(char *str)
86{
87 __nospectre_v2 = true;
88 return 0;
89}
90early_param("nospectre_v2", parse_spectre_v2_param);
91
92static bool spectre_v2_mitigations_off(void)
93{
94 return __nospectre_v2 || cpu_mitigations_off();
95}
96
97static const char *get_bhb_affected_string(enum mitigation_state bhb_state)
98{
99 switch (bhb_state) {
100 case SPECTRE_UNAFFECTED:
101 return "";
102 default:
103 case SPECTRE_VULNERABLE:
104 return ", but not BHB";
105 case SPECTRE_MITIGATED:
106 return ", BHB";
107 }
108}
109
110static bool _unprivileged_ebpf_enabled(void)
111{
112#ifdef CONFIG_BPF_SYSCALL
113 return !sysctl_unprivileged_bpf_disabled;
114#else
115 return false;
116#endif
117}
118
119ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
120 char *buf)
121{
122 enum mitigation_state bhb_state = arm64_get_spectre_bhb_state();
123 const char *bhb_str = get_bhb_affected_string(bhb_state: bhb_state);
124 const char *v2_str = "Branch predictor hardening";
125
126 switch (spectre_v2_state) {
127 case SPECTRE_UNAFFECTED:
128 if (bhb_state == SPECTRE_UNAFFECTED)
129 return sprintf(buf, fmt: "Not affected\n");
130
131 /*
132 * Platforms affected by Spectre-BHB can't report
133 * "Not affected" for Spectre-v2.
134 */
135 v2_str = "CSV2";
136 fallthrough;
137 case SPECTRE_MITIGATED:
138 if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled())
139 return sprintf(buf, fmt: "Vulnerable: Unprivileged eBPF enabled\n");
140
141 return sprintf(buf, fmt: "Mitigation: %s%s\n", v2_str, bhb_str);
142 case SPECTRE_VULNERABLE:
143 fallthrough;
144 default:
145 return sprintf(buf, fmt: "Vulnerable\n");
146 }
147}
148
149static enum mitigation_state spectre_v2_get_cpu_hw_mitigation_state(void)
150{
151 u64 pfr0;
152 static const struct midr_range spectre_v2_safe_list[] = {
153 MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
154 MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
155 MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
156 MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
157 MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
158 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
159 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
160 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
161 { /* sentinel */ }
162 };
163
164 /* If the CPU has CSV2 set, we're safe */
165 pfr0 = read_cpuid(ID_AA64PFR0_EL1);
166 if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_CSV2_SHIFT))
167 return SPECTRE_UNAFFECTED;
168
169 /* Alternatively, we have a list of unaffected CPUs */
170 if (is_midr_in_range_list(spectre_v2_safe_list))
171 return SPECTRE_UNAFFECTED;
172
173 return SPECTRE_VULNERABLE;
174}
175
176static enum mitigation_state spectre_v2_get_cpu_fw_mitigation_state(void)
177{
178 int ret;
179 struct arm_smccc_res res;
180
181 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
182 ARM_SMCCC_ARCH_WORKAROUND_1, &res);
183
184 ret = res.a0;
185 switch (ret) {
186 case SMCCC_RET_SUCCESS:
187 return SPECTRE_MITIGATED;
188 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
189 return SPECTRE_UNAFFECTED;
190 default:
191 fallthrough;
192 case SMCCC_RET_NOT_SUPPORTED:
193 return SPECTRE_VULNERABLE;
194 }
195}
196
197bool has_spectre_v2(const struct arm64_cpu_capabilities *entry, int scope)
198{
199 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
200
201 if (spectre_v2_get_cpu_hw_mitigation_state() == SPECTRE_UNAFFECTED)
202 return false;
203
204 if (spectre_v2_get_cpu_fw_mitigation_state() == SPECTRE_UNAFFECTED)
205 return false;
206
207 return true;
208}
209
210enum mitigation_state arm64_get_spectre_v2_state(void)
211{
212 return spectre_v2_state;
213}
214
215DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
216
217static void install_bp_hardening_cb(bp_hardening_cb_t fn)
218{
219 __this_cpu_write(bp_hardening_data.fn, fn);
220
221 /*
222 * Vinz Clortho takes the hyp_vecs start/end "keys" at
223 * the door when we're a guest. Skip the hyp-vectors work.
224 */
225 if (!is_hyp_mode_available())
226 return;
227
228 __this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT);
229}
230
231/* Called during entry so must be noinstr */
232static noinstr void call_smc_arch_workaround_1(void)
233{
234 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
235}
236
237/* Called during entry so must be noinstr */
238static noinstr void call_hvc_arch_workaround_1(void)
239{
240 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
241}
242
243/* Called during entry so must be noinstr */
244static noinstr void qcom_link_stack_sanitisation(void)
245{
246 u64 tmp;
247
248 asm volatile("mov %0, x30 \n"
249 ".rept 16 \n"
250 "bl . + 4 \n"
251 ".endr \n"
252 "mov x30, %0 \n"
253 : "=&r" (tmp));
254}
255
256static bp_hardening_cb_t spectre_v2_get_sw_mitigation_cb(void)
257{
258 u32 midr = read_cpuid_id();
259 if (((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR) &&
260 ((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR_V1))
261 return NULL;
262
263 return qcom_link_stack_sanitisation;
264}
265
266static enum mitigation_state spectre_v2_enable_fw_mitigation(void)
267{
268 bp_hardening_cb_t cb;
269 enum mitigation_state state;
270
271 state = spectre_v2_get_cpu_fw_mitigation_state();
272 if (state != SPECTRE_MITIGATED)
273 return state;
274
275 if (spectre_v2_mitigations_off())
276 return SPECTRE_VULNERABLE;
277
278 switch (arm_smccc_1_1_get_conduit()) {
279 case SMCCC_CONDUIT_HVC:
280 cb = call_hvc_arch_workaround_1;
281 break;
282
283 case SMCCC_CONDUIT_SMC:
284 cb = call_smc_arch_workaround_1;
285 break;
286
287 default:
288 return SPECTRE_VULNERABLE;
289 }
290
291 /*
292 * Prefer a CPU-specific workaround if it exists. Note that we
293 * still rely on firmware for the mitigation at EL2.
294 */
295 cb = spectre_v2_get_sw_mitigation_cb() ?: cb;
296 install_bp_hardening_cb(cb);
297 return SPECTRE_MITIGATED;
298}
299
300void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
301{
302 enum mitigation_state state;
303
304 WARN_ON(preemptible());
305
306 state = spectre_v2_get_cpu_hw_mitigation_state();
307 if (state == SPECTRE_VULNERABLE)
308 state = spectre_v2_enable_fw_mitigation();
309
310 update_mitigation_state(oldp: &spectre_v2_state, new: state);
311}
312
313/*
314 * Spectre-v3a.
315 *
316 * Phew, there's not an awful lot to do here! We just instruct EL2 to use
317 * an indirect trampoline for the hyp vectors so that guests can't read
318 * VBAR_EL2 to defeat randomisation of the hypervisor VA layout.
319 */
320bool has_spectre_v3a(const struct arm64_cpu_capabilities *entry, int scope)
321{
322 static const struct midr_range spectre_v3a_unsafe_list[] = {
323 MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
324 MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
325 {},
326 };
327
328 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
329 return is_midr_in_range_list(spectre_v3a_unsafe_list);
330}
331
332void spectre_v3a_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
333{
334 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
335
336 if (this_cpu_has_cap(ARM64_SPECTRE_V3A))
337 data->slot += HYP_VECTOR_INDIRECT;
338}
339
340/*
341 * Spectre v4.
342 *
343 * If you thought Spectre v2 was nasty, wait until you see this mess. A CPU is
344 * either:
345 *
346 * - Mitigated in hardware and listed in our "safe list".
347 * - Mitigated in hardware via PSTATE.SSBS.
348 * - Mitigated in software by firmware (sometimes referred to as SSBD).
349 *
350 * Wait, that doesn't sound so bad, does it? Keep reading...
351 *
352 * A major source of headaches is that the software mitigation is enabled both
353 * on a per-task basis, but can also be forced on for the kernel, necessitating
354 * both context-switch *and* entry/exit hooks. To make it even worse, some CPUs
355 * allow EL0 to toggle SSBS directly, which can end up with the prctl() state
356 * being stale when re-entering the kernel. The usual big.LITTLE caveats apply,
357 * so you can have systems that have both firmware and SSBS mitigations. This
358 * means we actually have to reject late onlining of CPUs with mitigations if
359 * all of the currently onlined CPUs are safelisted, as the mitigation tends to
360 * be opt-in for userspace. Yes, really, the cure is worse than the disease.
361 *
362 * The only good part is that if the firmware mitigation is present, then it is
363 * present for all CPUs, meaning we don't have to worry about late onlining of a
364 * vulnerable CPU if one of the boot CPUs is using the firmware mitigation.
365 *
366 * Give me a VAX-11/780 any day of the week...
367 */
368static enum mitigation_state spectre_v4_state;
369
370/* This is the per-cpu state tracking whether we need to talk to firmware */
371DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
372
373enum spectre_v4_policy {
374 SPECTRE_V4_POLICY_MITIGATION_DYNAMIC,
375 SPECTRE_V4_POLICY_MITIGATION_ENABLED,
376 SPECTRE_V4_POLICY_MITIGATION_DISABLED,
377};
378
379static enum spectre_v4_policy __read_mostly __spectre_v4_policy;
380
381static const struct spectre_v4_param {
382 const char *str;
383 enum spectre_v4_policy policy;
384} spectre_v4_params[] = {
385 { "force-on", SPECTRE_V4_POLICY_MITIGATION_ENABLED, },
386 { "force-off", SPECTRE_V4_POLICY_MITIGATION_DISABLED, },
387 { "kernel", SPECTRE_V4_POLICY_MITIGATION_DYNAMIC, },
388};
389static int __init parse_spectre_v4_param(char *str)
390{
391 int i;
392
393 if (!str || !str[0])
394 return -EINVAL;
395
396 for (i = 0; i < ARRAY_SIZE(spectre_v4_params); i++) {
397 const struct spectre_v4_param *param = &spectre_v4_params[i];
398
399 if (strncmp(str, param->str, strlen(param->str)))
400 continue;
401
402 __spectre_v4_policy = param->policy;
403 return 0;
404 }
405
406 return -EINVAL;
407}
408early_param("ssbd", parse_spectre_v4_param);
409
410/*
411 * Because this was all written in a rush by people working in different silos,
412 * we've ended up with multiple command line options to control the same thing.
413 * Wrap these up in some helpers, which prefer disabling the mitigation if faced
414 * with contradictory parameters. The mitigation is always either "off",
415 * "dynamic" or "on".
416 */
417static bool spectre_v4_mitigations_off(void)
418{
419 return cpu_mitigations_off() ||
420 __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DISABLED;
421}
422
423/* Do we need to toggle the mitigation state on entry to/exit from the kernel? */
424static bool spectre_v4_mitigations_dynamic(void)
425{
426 return !spectre_v4_mitigations_off() &&
427 __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DYNAMIC;
428}
429
430static bool spectre_v4_mitigations_on(void)
431{
432 return !spectre_v4_mitigations_off() &&
433 __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_ENABLED;
434}
435
436ssize_t cpu_show_spec_store_bypass(struct device *dev,
437 struct device_attribute *attr, char *buf)
438{
439 switch (spectre_v4_state) {
440 case SPECTRE_UNAFFECTED:
441 return sprintf(buf, fmt: "Not affected\n");
442 case SPECTRE_MITIGATED:
443 return sprintf(buf, fmt: "Mitigation: Speculative Store Bypass disabled via prctl\n");
444 case SPECTRE_VULNERABLE:
445 fallthrough;
446 default:
447 return sprintf(buf, fmt: "Vulnerable\n");
448 }
449}
450
451enum mitigation_state arm64_get_spectre_v4_state(void)
452{
453 return spectre_v4_state;
454}
455
456static enum mitigation_state spectre_v4_get_cpu_hw_mitigation_state(void)
457{
458 static const struct midr_range spectre_v4_safe_list[] = {
459 MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
460 MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
461 MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
462 MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
463 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
464 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
465 { /* sentinel */ },
466 };
467
468 if (is_midr_in_range_list(spectre_v4_safe_list))
469 return SPECTRE_UNAFFECTED;
470
471 /* CPU features are detected first */
472 if (this_cpu_has_cap(ARM64_SSBS))
473 return SPECTRE_MITIGATED;
474
475 return SPECTRE_VULNERABLE;
476}
477
478static enum mitigation_state spectre_v4_get_cpu_fw_mitigation_state(void)
479{
480 int ret;
481 struct arm_smccc_res res;
482
483 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
484 ARM_SMCCC_ARCH_WORKAROUND_2, &res);
485
486 ret = res.a0;
487 switch (ret) {
488 case SMCCC_RET_SUCCESS:
489 return SPECTRE_MITIGATED;
490 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
491 fallthrough;
492 case SMCCC_RET_NOT_REQUIRED:
493 return SPECTRE_UNAFFECTED;
494 default:
495 fallthrough;
496 case SMCCC_RET_NOT_SUPPORTED:
497 return SPECTRE_VULNERABLE;
498 }
499}
500
501bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope)
502{
503 enum mitigation_state state;
504
505 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
506
507 state = spectre_v4_get_cpu_hw_mitigation_state();
508 if (state == SPECTRE_VULNERABLE)
509 state = spectre_v4_get_cpu_fw_mitigation_state();
510
511 return state != SPECTRE_UNAFFECTED;
512}
513
514bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr)
515{
516 const u32 instr_mask = ~(1U << PSTATE_Imm_shift);
517 const u32 instr_val = 0xd500401f | PSTATE_SSBS;
518
519 if ((instr & instr_mask) != instr_val)
520 return false;
521
522 if (instr & BIT(PSTATE_Imm_shift))
523 regs->pstate |= PSR_SSBS_BIT;
524 else
525 regs->pstate &= ~PSR_SSBS_BIT;
526
527 arm64_skip_faulting_instruction(regs, 4);
528 return true;
529}
530
531static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
532{
533 enum mitigation_state state;
534
535 /*
536 * If the system is mitigated but this CPU doesn't have SSBS, then
537 * we must be on the safelist and there's nothing more to do.
538 */
539 state = spectre_v4_get_cpu_hw_mitigation_state();
540 if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS))
541 return state;
542
543 if (spectre_v4_mitigations_off()) {
544 sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
545 set_pstate_ssbs(1);
546 return SPECTRE_VULNERABLE;
547 }
548
549 /* SCTLR_EL1.DSSBS was initialised to 0 during boot */
550 set_pstate_ssbs(0);
551
552 /*
553 * SSBS is self-synchronizing and is intended to affect subsequent
554 * speculative instructions, but some CPUs can speculate with a stale
555 * value of SSBS.
556 *
557 * Mitigate this with an unconditional speculation barrier, as CPUs
558 * could mis-speculate branches and bypass a conditional barrier.
559 */
560 if (IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386))
561 spec_bar();
562
563 return SPECTRE_MITIGATED;
564}
565
566/*
567 * Patch a branch over the Spectre-v4 mitigation code with a NOP so that
568 * we fallthrough and check whether firmware needs to be called on this CPU.
569 */
570void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt,
571 __le32 *origptr,
572 __le32 *updptr, int nr_inst)
573{
574 BUG_ON(nr_inst != 1); /* Branch -> NOP */
575
576 if (spectre_v4_mitigations_off())
577 return;
578
579 if (cpus_have_cap(ARM64_SSBS))
580 return;
581
582 if (spectre_v4_mitigations_dynamic())
583 *updptr = cpu_to_le32(aarch64_insn_gen_nop());
584}
585
586/*
587 * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction
588 * to call into firmware to adjust the mitigation state.
589 */
590void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt,
591 __le32 *origptr,
592 __le32 *updptr, int nr_inst)
593{
594 u32 insn;
595
596 BUG_ON(nr_inst != 1); /* NOP -> HVC/SMC */
597
598 switch (arm_smccc_1_1_get_conduit()) {
599 case SMCCC_CONDUIT_HVC:
600 insn = aarch64_insn_get_hvc_value();
601 break;
602 case SMCCC_CONDUIT_SMC:
603 insn = aarch64_insn_get_smc_value();
604 break;
605 default:
606 return;
607 }
608
609 *updptr = cpu_to_le32(insn);
610}
611
612static enum mitigation_state spectre_v4_enable_fw_mitigation(void)
613{
614 enum mitigation_state state;
615
616 state = spectre_v4_get_cpu_fw_mitigation_state();
617 if (state != SPECTRE_MITIGATED)
618 return state;
619
620 if (spectre_v4_mitigations_off()) {
621 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, false, NULL);
622 return SPECTRE_VULNERABLE;
623 }
624
625 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, true, NULL);
626
627 if (spectre_v4_mitigations_dynamic())
628 __this_cpu_write(arm64_ssbd_callback_required, 1);
629
630 return SPECTRE_MITIGATED;
631}
632
633void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
634{
635 enum mitigation_state state;
636
637 WARN_ON(preemptible());
638
639 state = spectre_v4_enable_hw_mitigation();
640 if (state == SPECTRE_VULNERABLE)
641 state = spectre_v4_enable_fw_mitigation();
642
643 update_mitigation_state(oldp: &spectre_v4_state, new: state);
644}
645
646static void __update_pstate_ssbs(struct pt_regs *regs, bool state)
647{
648 u64 bit = compat_user_mode(regs) ? PSR_AA32_SSBS_BIT : PSR_SSBS_BIT;
649
650 if (state)
651 regs->pstate |= bit;
652 else
653 regs->pstate &= ~bit;
654}
655
656void spectre_v4_enable_task_mitigation(struct task_struct *tsk)
657{
658 struct pt_regs *regs = task_pt_regs(tsk);
659 bool ssbs = false, kthread = tsk->flags & PF_KTHREAD;
660
661 if (spectre_v4_mitigations_off())
662 ssbs = true;
663 else if (spectre_v4_mitigations_dynamic() && !kthread)
664 ssbs = !test_tsk_thread_flag(tsk, TIF_SSBD);
665
666 __update_pstate_ssbs(regs, state: ssbs);
667}
668
669/*
670 * The Spectre-v4 mitigation can be controlled via a prctl() from userspace.
671 * This is interesting because the "speculation disabled" behaviour can be
672 * configured so that it is preserved across exec(), which means that the
673 * prctl() may be necessary even when PSTATE.SSBS can be toggled directly
674 * from userspace.
675 */
676static void ssbd_prctl_enable_mitigation(struct task_struct *task)
677{
678 task_clear_spec_ssb_noexec(p: task);
679 task_set_spec_ssb_disable(p: task);
680 set_tsk_thread_flag(tsk: task, TIF_SSBD);
681}
682
683static void ssbd_prctl_disable_mitigation(struct task_struct *task)
684{
685 task_clear_spec_ssb_noexec(p: task);
686 task_clear_spec_ssb_disable(p: task);
687 clear_tsk_thread_flag(tsk: task, TIF_SSBD);
688}
689
690static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)
691{
692 switch (ctrl) {
693 case PR_SPEC_ENABLE:
694 /* Enable speculation: disable mitigation */
695 /*
696 * Force disabled speculation prevents it from being
697 * re-enabled.
698 */
699 if (task_spec_ssb_force_disable(p: task))
700 return -EPERM;
701
702 /*
703 * If the mitigation is forced on, then speculation is forced
704 * off and we again prevent it from being re-enabled.
705 */
706 if (spectre_v4_mitigations_on())
707 return -EPERM;
708
709 ssbd_prctl_disable_mitigation(task);
710 break;
711 case PR_SPEC_FORCE_DISABLE:
712 /* Force disable speculation: force enable mitigation */
713 /*
714 * If the mitigation is forced off, then speculation is forced
715 * on and we prevent it from being disabled.
716 */
717 if (spectre_v4_mitigations_off())
718 return -EPERM;
719
720 task_set_spec_ssb_force_disable(p: task);
721 fallthrough;
722 case PR_SPEC_DISABLE:
723 /* Disable speculation: enable mitigation */
724 /* Same as PR_SPEC_FORCE_DISABLE */
725 if (spectre_v4_mitigations_off())
726 return -EPERM;
727
728 ssbd_prctl_enable_mitigation(task);
729 break;
730 case PR_SPEC_DISABLE_NOEXEC:
731 /* Disable speculation until execve(): enable mitigation */
732 /*
733 * If the mitigation state is forced one way or the other, then
734 * we must fail now before we try to toggle it on execve().
735 */
736 if (task_spec_ssb_force_disable(p: task) ||
737 spectre_v4_mitigations_off() ||
738 spectre_v4_mitigations_on()) {
739 return -EPERM;
740 }
741
742 ssbd_prctl_enable_mitigation(task);
743 task_set_spec_ssb_noexec(p: task);
744 break;
745 default:
746 return -ERANGE;
747 }
748
749 spectre_v4_enable_task_mitigation(tsk: task);
750 return 0;
751}
752
753int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
754 unsigned long ctrl)
755{
756 switch (which) {
757 case PR_SPEC_STORE_BYPASS:
758 return ssbd_prctl_set(task, ctrl);
759 default:
760 return -ENODEV;
761 }
762}
763
764static int ssbd_prctl_get(struct task_struct *task)
765{
766 switch (spectre_v4_state) {
767 case SPECTRE_UNAFFECTED:
768 return PR_SPEC_NOT_AFFECTED;
769 case SPECTRE_MITIGATED:
770 if (spectre_v4_mitigations_on())
771 return PR_SPEC_NOT_AFFECTED;
772
773 if (spectre_v4_mitigations_dynamic())
774 break;
775
776 /* Mitigations are disabled, so we're vulnerable. */
777 fallthrough;
778 case SPECTRE_VULNERABLE:
779 fallthrough;
780 default:
781 return PR_SPEC_ENABLE;
782 }
783
784 /* Check the mitigation state for this task */
785 if (task_spec_ssb_force_disable(p: task))
786 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
787
788 if (task_spec_ssb_noexec(p: task))
789 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
790
791 if (task_spec_ssb_disable(p: task))
792 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
793
794 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
795}
796
797int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
798{
799 switch (which) {
800 case PR_SPEC_STORE_BYPASS:
801 return ssbd_prctl_get(task);
802 default:
803 return -ENODEV;
804 }
805}
806
807/*
808 * Spectre BHB.
809 *
810 * A CPU is either:
811 * - Mitigated by a branchy loop a CPU specific number of times, and listed
812 * in our "loop mitigated list".
813 * - Mitigated in software by the firmware Spectre v2 call.
814 * - Has the ClearBHB instruction to perform the mitigation.
815 * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
816 * software mitigation in the vectors is needed.
817 * - Has CSV2.3, so is unaffected.
818 */
819static enum mitigation_state spectre_bhb_state;
820
821enum mitigation_state arm64_get_spectre_bhb_state(void)
822{
823 return spectre_bhb_state;
824}
825
826enum bhb_mitigation_bits {
827 BHB_LOOP,
828 BHB_FW,
829 BHB_HW,
830 BHB_INSN,
831};
832static unsigned long system_bhb_mitigations;
833
834/*
835 * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
836 * SCOPE_SYSTEM call will give the right answer.
837 */
838static bool is_spectre_bhb_safe(int scope)
839{
840 static const struct midr_range spectre_bhb_safe_list[] = {
841 MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
842 MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
843 MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
844 MIDR_ALL_VERSIONS(MIDR_CORTEX_A510),
845 MIDR_ALL_VERSIONS(MIDR_CORTEX_A520),
846 MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
847 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
848 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
849 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
850 {},
851 };
852 static bool all_safe = true;
853
854 if (scope != SCOPE_LOCAL_CPU)
855 return all_safe;
856
857 if (is_midr_in_range_list(spectre_bhb_safe_list))
858 return true;
859
860 all_safe = false;
861
862 return false;
863}
864
865static u8 spectre_bhb_loop_affected(void)
866{
867 u8 k = 0;
868
869 static const struct midr_range spectre_bhb_k132_list[] = {
870 MIDR_ALL_VERSIONS(MIDR_CORTEX_X3),
871 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2),
872 {},
873 };
874 static const struct midr_range spectre_bhb_k38_list[] = {
875 MIDR_ALL_VERSIONS(MIDR_CORTEX_A715),
876 MIDR_ALL_VERSIONS(MIDR_CORTEX_A720),
877 MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE),
878 {},
879 };
880 static const struct midr_range spectre_bhb_k32_list[] = {
881 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
882 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE),
883 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
884 MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
885 MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C),
886 MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
887 MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
888 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
889 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
890 {},
891 };
892 static const struct midr_range spectre_bhb_k24_list[] = {
893 MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
894 MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE),
895 MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
896 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
897 MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_GOLD),
898 MIDR_ALL_VERSIONS(MIDR_HISI_HIP09),
899 {},
900 };
901 static const struct midr_range spectre_bhb_k11_list[] = {
902 MIDR_ALL_VERSIONS(MIDR_AMPERE1),
903 {},
904 };
905 static const struct midr_range spectre_bhb_k8_list[] = {
906 MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
907 MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
908 {},
909 };
910
911 if (is_midr_in_range_list(spectre_bhb_k132_list))
912 k = 132;
913 else if (is_midr_in_range_list(spectre_bhb_k38_list))
914 k = 38;
915 else if (is_midr_in_range_list(spectre_bhb_k32_list))
916 k = 32;
917 else if (is_midr_in_range_list(spectre_bhb_k24_list))
918 k = 24;
919 else if (is_midr_in_range_list(spectre_bhb_k11_list))
920 k = 11;
921 else if (is_midr_in_range_list(spectre_bhb_k8_list))
922 k = 8;
923
924 return k;
925}
926
927static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void)
928{
929 int ret;
930 struct arm_smccc_res res;
931
932 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
933 ARM_SMCCC_ARCH_WORKAROUND_3, &res);
934
935 ret = res.a0;
936 switch (ret) {
937 case SMCCC_RET_SUCCESS:
938 return SPECTRE_MITIGATED;
939 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
940 return SPECTRE_UNAFFECTED;
941 default:
942 fallthrough;
943 case SMCCC_RET_NOT_SUPPORTED:
944 return SPECTRE_VULNERABLE;
945 }
946}
947
948static bool has_spectre_bhb_fw_mitigation(void)
949{
950 enum mitigation_state fw_state;
951 bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE;
952
953 fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
954 return has_smccc && fw_state == SPECTRE_MITIGATED;
955}
956
957static bool supports_ecbhb(int scope)
958{
959 u64 mmfr1;
960
961 if (scope == SCOPE_LOCAL_CPU)
962 mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1);
963 else
964 mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
965
966 return cpuid_feature_extract_unsigned_field(mmfr1,
967 ID_AA64MMFR1_EL1_ECBHB_SHIFT);
968}
969
970static u8 max_bhb_k;
971
972bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
973 int scope)
974{
975 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
976
977 if (supports_csv2p3(scope))
978 return false;
979
980 if (is_spectre_bhb_safe(scope))
981 return false;
982
983 /*
984 * At this point the core isn't known to be "safe" so we're going to
985 * assume it's vulnerable. We still need to update `max_bhb_k` though,
986 * but only if we aren't mitigating with clearbhb though.
987 */
988 if (scope == SCOPE_LOCAL_CPU && !supports_clearbhb(SCOPE_LOCAL_CPU))
989 max_bhb_k = max(max_bhb_k, spectre_bhb_loop_affected());
990
991 return true;
992}
993
994u8 get_spectre_bhb_loop_value(void)
995{
996 return max_bhb_k;
997}
998
999static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
1000{
1001 const char *v = arm64_get_bp_hardening_vector(slot);
1002
1003 __this_cpu_write(this_cpu_vector, v);
1004
1005 /*
1006 * When KPTI is in use, the vectors are switched when exiting to
1007 * user-space.
1008 */
1009 if (cpus_have_cap(ARM64_UNMAP_KERNEL_AT_EL0))
1010 return;
1011
1012 write_sysreg(v, vbar_el1);
1013 isb();
1014}
1015
1016bool __read_mostly __nospectre_bhb;
1017static int __init parse_spectre_bhb_param(char *str)
1018{
1019 __nospectre_bhb = true;
1020 return 0;
1021}
1022early_param("nospectre_bhb", parse_spectre_bhb_param);
1023
1024void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
1025{
1026 bp_hardening_cb_t cpu_cb;
1027 enum mitigation_state state = SPECTRE_VULNERABLE;
1028 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
1029
1030 if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU))
1031 return;
1032
1033 if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) {
1034 /* No point mitigating Spectre-BHB alone. */
1035 } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
1036 /* Do nothing */
1037 } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
1038 state = SPECTRE_MITIGATED;
1039 set_bit(nr: BHB_HW, addr: &system_bhb_mitigations);
1040 } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
1041 /*
1042 * Ensure KVM uses the indirect vector which will have ClearBHB
1043 * added.
1044 */
1045 if (!data->slot)
1046 data->slot = HYP_VECTOR_INDIRECT;
1047
1048 this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
1049 state = SPECTRE_MITIGATED;
1050 set_bit(nr: BHB_INSN, addr: &system_bhb_mitigations);
1051 } else if (spectre_bhb_loop_affected()) {
1052 /*
1053 * Ensure KVM uses the indirect vector which will have the
1054 * branchy-loop added. A57/A72-r0 will already have selected
1055 * the spectre-indirect vector, which is sufficient for BHB
1056 * too.
1057 */
1058 if (!data->slot)
1059 data->slot = HYP_VECTOR_INDIRECT;
1060
1061 this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP);
1062 state = SPECTRE_MITIGATED;
1063 set_bit(nr: BHB_LOOP, addr: &system_bhb_mitigations);
1064 } else if (has_spectre_bhb_fw_mitigation()) {
1065 /*
1066 * Ensure KVM uses one of the spectre bp_hardening
1067 * vectors. The indirect vector doesn't include the EL3
1068 * call, so needs upgrading to
1069 * HYP_VECTOR_SPECTRE_INDIRECT.
1070 */
1071 if (!data->slot || data->slot == HYP_VECTOR_INDIRECT)
1072 data->slot += 1;
1073
1074 this_cpu_set_vectors(EL1_VECTOR_BHB_FW);
1075
1076 /*
1077 * The WA3 call in the vectors supersedes the WA1 call
1078 * made during context-switch. Uninstall any firmware
1079 * bp_hardening callback.
1080 */
1081 cpu_cb = spectre_v2_get_sw_mitigation_cb();
1082 if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb)
1083 __this_cpu_write(bp_hardening_data.fn, NULL);
1084
1085 state = SPECTRE_MITIGATED;
1086 set_bit(nr: BHB_FW, addr: &system_bhb_mitigations);
1087 }
1088
1089 update_mitigation_state(oldp: &spectre_bhb_state, new: state);
1090}
1091
1092bool is_spectre_bhb_fw_mitigated(void)
1093{
1094 return test_bit(BHB_FW, &system_bhb_mitigations);
1095}
1096
1097/* Patched to NOP when enabled */
1098void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt,
1099 __le32 *origptr,
1100 __le32 *updptr, int nr_inst)
1101{
1102 BUG_ON(nr_inst != 1);
1103
1104 if (test_bit(BHB_LOOP, &system_bhb_mitigations))
1105 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1106}
1107
1108/* Patched to NOP when enabled */
1109void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt,
1110 __le32 *origptr,
1111 __le32 *updptr, int nr_inst)
1112{
1113 BUG_ON(nr_inst != 1);
1114
1115 if (test_bit(BHB_FW, &system_bhb_mitigations))
1116 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1117}
1118
1119/* Patched to correct the immediate */
1120void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt,
1121 __le32 *origptr, __le32 *updptr, int nr_inst)
1122{
1123 u8 rd;
1124 u32 insn;
1125
1126 BUG_ON(nr_inst != 1); /* MOV -> MOV */
1127
1128 if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY))
1129 return;
1130
1131 insn = le32_to_cpu(*origptr);
1132 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
1133 insn = aarch64_insn_gen_movewide(rd, max_bhb_k, 0,
1134 AARCH64_INSN_VARIANT_64BIT,
1135 AARCH64_INSN_MOVEWIDE_ZERO);
1136 *updptr++ = cpu_to_le32(insn);
1137}
1138
1139/* Patched to mov WA3 when supported */
1140void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt,
1141 __le32 *origptr, __le32 *updptr, int nr_inst)
1142{
1143 u8 rd;
1144 u32 insn;
1145
1146 BUG_ON(nr_inst != 1); /* MOV -> MOV */
1147
1148 if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) ||
1149 !test_bit(BHB_FW, &system_bhb_mitigations))
1150 return;
1151
1152 insn = le32_to_cpu(*origptr);
1153 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
1154
1155 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR,
1156 AARCH64_INSN_VARIANT_32BIT,
1157 AARCH64_INSN_REG_ZR, rd,
1158 ARM_SMCCC_ARCH_WORKAROUND_3);
1159 if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT))
1160 return;
1161
1162 *updptr++ = cpu_to_le32(insn);
1163}
1164
1165/* Patched to NOP when not supported */
1166void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt,
1167 __le32 *origptr, __le32 *updptr, int nr_inst)
1168{
1169 BUG_ON(nr_inst != 2);
1170
1171 if (test_bit(BHB_INSN, &system_bhb_mitigations))
1172 return;
1173
1174 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1175 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1176}
1177
1178#ifdef CONFIG_BPF_SYSCALL
1179#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n"
1180void unpriv_ebpf_notify(int new_state)
1181{
1182 if (spectre_v2_state == SPECTRE_VULNERABLE ||
1183 spectre_bhb_state != SPECTRE_MITIGATED)
1184 return;
1185
1186 if (!new_state)
1187 pr_err("WARNING: %s", EBPF_WARN);
1188}
1189#endif
1190
1191void spectre_print_disabled_mitigations(void)
1192{
1193 /* Keep a single copy of the common message suffix to avoid duplication. */
1194 const char *spectre_disabled_suffix = "mitigation disabled by command-line option\n";
1195
1196 if (spectre_v2_mitigations_off())
1197 pr_info("spectre-v2 %s", spectre_disabled_suffix);
1198
1199 if (spectre_v4_mitigations_off())
1200 pr_info("spectre-v4 %s", spectre_disabled_suffix);
1201
1202 if (__nospectre_bhb || cpu_mitigations_off())
1203 pr_info("spectre-bhb %s", spectre_disabled_suffix);
1204}
1205

source code of linux/arch/arm64/kernel/proton-pack.c