1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 1994 Linus Torvalds |
4 | * |
5 | * Cyrix stuff, June 1998 by: |
6 | * - Rafael R. Reilova (moved everything from head.S), |
7 | * <rreilova@ececs.uc.edu> |
8 | * - Channing Corn (tests & fixes), |
9 | * - Andrew D. Balsa (code cleanup). |
10 | */ |
11 | #include <linux/init.h> |
12 | #include <linux/cpu.h> |
13 | #include <linux/module.h> |
14 | #include <linux/nospec.h> |
15 | #include <linux/prctl.h> |
16 | #include <linux/sched/smt.h> |
17 | #include <linux/pgtable.h> |
18 | #include <linux/bpf.h> |
19 | |
20 | #include <asm/spec-ctrl.h> |
21 | #include <asm/cmdline.h> |
22 | #include <asm/bugs.h> |
23 | #include <asm/processor.h> |
24 | #include <asm/processor-flags.h> |
25 | #include <asm/fpu/api.h> |
26 | #include <asm/msr.h> |
27 | #include <asm/vmx.h> |
28 | #include <asm/paravirt.h> |
29 | #include <asm/intel-family.h> |
30 | #include <asm/e820/api.h> |
31 | #include <asm/hypervisor.h> |
32 | #include <asm/tlbflush.h> |
33 | #include <asm/cpu.h> |
34 | |
35 | #include "cpu.h" |
36 | |
37 | static void __init spectre_v1_select_mitigation(void); |
38 | static void __init spectre_v2_select_mitigation(void); |
39 | static void __init retbleed_select_mitigation(void); |
40 | static void __init spectre_v2_user_select_mitigation(void); |
41 | static void __init ssb_select_mitigation(void); |
42 | static void __init l1tf_select_mitigation(void); |
43 | static void __init mds_select_mitigation(void); |
44 | static void __init md_clear_update_mitigation(void); |
45 | static void __init md_clear_select_mitigation(void); |
46 | static void __init taa_select_mitigation(void); |
47 | static void __init mmio_select_mitigation(void); |
48 | static void __init srbds_select_mitigation(void); |
49 | static void __init l1d_flush_select_mitigation(void); |
50 | static void __init srso_select_mitigation(void); |
51 | static void __init gds_select_mitigation(void); |
52 | |
53 | /* The base value of the SPEC_CTRL MSR without task-specific bits set */ |
54 | u64 x86_spec_ctrl_base; |
55 | EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); |
56 | |
57 | /* The current value of the SPEC_CTRL MSR with task-specific bits set */ |
58 | DEFINE_PER_CPU(u64, x86_spec_ctrl_current); |
59 | EXPORT_SYMBOL_GPL(x86_spec_ctrl_current); |
60 | |
61 | u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB; |
62 | EXPORT_SYMBOL_GPL(x86_pred_cmd); |
63 | |
64 | static DEFINE_MUTEX(spec_ctrl_mutex); |
65 | |
66 | void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk; |
67 | |
68 | /* Update SPEC_CTRL MSR and its cached copy unconditionally */ |
69 | static void update_spec_ctrl(u64 val) |
70 | { |
71 | this_cpu_write(x86_spec_ctrl_current, val); |
72 | wrmsrl(MSR_IA32_SPEC_CTRL, val); |
73 | } |
74 | |
75 | /* |
76 | * Keep track of the SPEC_CTRL MSR value for the current task, which may differ |
77 | * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). |
78 | */ |
79 | void update_spec_ctrl_cond(u64 val) |
80 | { |
81 | if (this_cpu_read(x86_spec_ctrl_current) == val) |
82 | return; |
83 | |
84 | this_cpu_write(x86_spec_ctrl_current, val); |
85 | |
86 | /* |
87 | * When KERNEL_IBRS this MSR is written on return-to-user, unless |
88 | * forced the update can be delayed until that time. |
89 | */ |
90 | if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) |
91 | wrmsrl(MSR_IA32_SPEC_CTRL, val); |
92 | } |
93 | |
94 | noinstr u64 spec_ctrl_current(void) |
95 | { |
96 | return this_cpu_read(x86_spec_ctrl_current); |
97 | } |
98 | EXPORT_SYMBOL_GPL(spec_ctrl_current); |
99 | |
100 | /* |
101 | * AMD specific MSR info for Speculative Store Bypass control. |
102 | * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). |
103 | */ |
104 | u64 __ro_after_init x86_amd_ls_cfg_base; |
105 | u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; |
106 | |
107 | /* Control conditional STIBP in switch_to() */ |
108 | DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); |
109 | /* Control conditional IBPB in switch_mm() */ |
110 | DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); |
111 | /* Control unconditional IBPB in switch_mm() */ |
112 | DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); |
113 | |
114 | /* Control MDS CPU buffer clear before returning to user space */ |
115 | DEFINE_STATIC_KEY_FALSE(mds_user_clear); |
116 | EXPORT_SYMBOL_GPL(mds_user_clear); |
117 | /* Control MDS CPU buffer clear before idling (halt, mwait) */ |
118 | DEFINE_STATIC_KEY_FALSE(mds_idle_clear); |
119 | EXPORT_SYMBOL_GPL(mds_idle_clear); |
120 | |
121 | /* |
122 | * Controls whether l1d flush based mitigations are enabled, |
123 | * based on hw features and admin setting via boot parameter |
124 | * defaults to false |
125 | */ |
126 | DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); |
127 | |
128 | /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */ |
129 | DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear); |
130 | EXPORT_SYMBOL_GPL(mmio_stale_data_clear); |
131 | |
132 | void __init cpu_select_mitigations(void) |
133 | { |
134 | /* |
135 | * Read the SPEC_CTRL MSR to account for reserved bits which may |
136 | * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD |
137 | * init code as it is not enumerated and depends on the family. |
138 | */ |
139 | if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { |
140 | rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); |
141 | |
142 | /* |
143 | * Previously running kernel (kexec), may have some controls |
144 | * turned ON. Clear them and let the mitigations setup below |
145 | * rediscover them based on configuration. |
146 | */ |
147 | x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK; |
148 | } |
149 | |
150 | /* Select the proper CPU mitigations before patching alternatives: */ |
151 | spectre_v1_select_mitigation(); |
152 | spectre_v2_select_mitigation(); |
153 | /* |
154 | * retbleed_select_mitigation() relies on the state set by |
155 | * spectre_v2_select_mitigation(); specifically it wants to know about |
156 | * spectre_v2=ibrs. |
157 | */ |
158 | retbleed_select_mitigation(); |
159 | /* |
160 | * spectre_v2_user_select_mitigation() relies on the state set by |
161 | * retbleed_select_mitigation(); specifically the STIBP selection is |
162 | * forced for UNRET or IBPB. |
163 | */ |
164 | spectre_v2_user_select_mitigation(); |
165 | ssb_select_mitigation(); |
166 | l1tf_select_mitigation(); |
167 | md_clear_select_mitigation(); |
168 | srbds_select_mitigation(); |
169 | l1d_flush_select_mitigation(); |
170 | |
171 | /* |
172 | * srso_select_mitigation() depends and must run after |
173 | * retbleed_select_mitigation(). |
174 | */ |
175 | srso_select_mitigation(); |
176 | gds_select_mitigation(); |
177 | } |
178 | |
179 | /* |
180 | * NOTE: This function is *only* called for SVM, since Intel uses |
181 | * MSR_IA32_SPEC_CTRL for SSBD. |
182 | */ |
183 | void |
184 | x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) |
185 | { |
186 | u64 guestval, hostval; |
187 | struct thread_info *ti = current_thread_info(); |
188 | |
189 | /* |
190 | * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update |
191 | * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. |
192 | */ |
193 | if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && |
194 | !static_cpu_has(X86_FEATURE_VIRT_SSBD)) |
195 | return; |
196 | |
197 | /* |
198 | * If the host has SSBD mitigation enabled, force it in the host's |
199 | * virtual MSR value. If its not permanently enabled, evaluate |
200 | * current's TIF_SSBD thread flag. |
201 | */ |
202 | if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) |
203 | hostval = SPEC_CTRL_SSBD; |
204 | else |
205 | hostval = ssbd_tif_to_spec_ctrl(tifn: ti->flags); |
206 | |
207 | /* Sanitize the guest value */ |
208 | guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; |
209 | |
210 | if (hostval != guestval) { |
211 | unsigned long tif; |
212 | |
213 | tif = setguest ? ssbd_spec_ctrl_to_tif(spec_ctrl: guestval) : |
214 | ssbd_spec_ctrl_to_tif(spec_ctrl: hostval); |
215 | |
216 | speculation_ctrl_update(tif); |
217 | } |
218 | } |
219 | EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); |
220 | |
221 | static void x86_amd_ssb_disable(void) |
222 | { |
223 | u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; |
224 | |
225 | if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) |
226 | wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); |
227 | else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) |
228 | wrmsrl(MSR_AMD64_LS_CFG, val: msrval); |
229 | } |
230 | |
231 | #undef pr_fmt |
232 | #define pr_fmt(fmt) "MDS: " fmt |
233 | |
234 | /* Default mitigation for MDS-affected CPUs */ |
235 | static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL; |
236 | static bool mds_nosmt __ro_after_init = false; |
237 | |
238 | static const char * const mds_strings[] = { |
239 | [MDS_MITIGATION_OFF] = "Vulnerable" , |
240 | [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers" , |
241 | [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode" , |
242 | }; |
243 | |
244 | static void __init mds_select_mitigation(void) |
245 | { |
246 | if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { |
247 | mds_mitigation = MDS_MITIGATION_OFF; |
248 | return; |
249 | } |
250 | |
251 | if (mds_mitigation == MDS_MITIGATION_FULL) { |
252 | if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) |
253 | mds_mitigation = MDS_MITIGATION_VMWERV; |
254 | |
255 | static_branch_enable(&mds_user_clear); |
256 | |
257 | if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && |
258 | (mds_nosmt || cpu_mitigations_auto_nosmt())) |
259 | cpu_smt_disable(force: false); |
260 | } |
261 | } |
262 | |
263 | static int __init mds_cmdline(char *str) |
264 | { |
265 | if (!boot_cpu_has_bug(X86_BUG_MDS)) |
266 | return 0; |
267 | |
268 | if (!str) |
269 | return -EINVAL; |
270 | |
271 | if (!strcmp(str, "off" )) |
272 | mds_mitigation = MDS_MITIGATION_OFF; |
273 | else if (!strcmp(str, "full" )) |
274 | mds_mitigation = MDS_MITIGATION_FULL; |
275 | else if (!strcmp(str, "full,nosmt" )) { |
276 | mds_mitigation = MDS_MITIGATION_FULL; |
277 | mds_nosmt = true; |
278 | } |
279 | |
280 | return 0; |
281 | } |
282 | early_param("mds" , mds_cmdline); |
283 | |
284 | #undef pr_fmt |
285 | #define pr_fmt(fmt) "TAA: " fmt |
286 | |
287 | enum taa_mitigations { |
288 | TAA_MITIGATION_OFF, |
289 | TAA_MITIGATION_UCODE_NEEDED, |
290 | TAA_MITIGATION_VERW, |
291 | TAA_MITIGATION_TSX_DISABLED, |
292 | }; |
293 | |
294 | /* Default mitigation for TAA-affected CPUs */ |
295 | static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW; |
296 | static bool taa_nosmt __ro_after_init; |
297 | |
298 | static const char * const taa_strings[] = { |
299 | [TAA_MITIGATION_OFF] = "Vulnerable" , |
300 | [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode" , |
301 | [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers" , |
302 | [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled" , |
303 | }; |
304 | |
305 | static void __init taa_select_mitigation(void) |
306 | { |
307 | u64 ia32_cap; |
308 | |
309 | if (!boot_cpu_has_bug(X86_BUG_TAA)) { |
310 | taa_mitigation = TAA_MITIGATION_OFF; |
311 | return; |
312 | } |
313 | |
314 | /* TSX previously disabled by tsx=off */ |
315 | if (!boot_cpu_has(X86_FEATURE_RTM)) { |
316 | taa_mitigation = TAA_MITIGATION_TSX_DISABLED; |
317 | return; |
318 | } |
319 | |
320 | if (cpu_mitigations_off()) { |
321 | taa_mitigation = TAA_MITIGATION_OFF; |
322 | return; |
323 | } |
324 | |
325 | /* |
326 | * TAA mitigation via VERW is turned off if both |
327 | * tsx_async_abort=off and mds=off are specified. |
328 | */ |
329 | if (taa_mitigation == TAA_MITIGATION_OFF && |
330 | mds_mitigation == MDS_MITIGATION_OFF) |
331 | return; |
332 | |
333 | if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) |
334 | taa_mitigation = TAA_MITIGATION_VERW; |
335 | else |
336 | taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; |
337 | |
338 | /* |
339 | * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. |
340 | * A microcode update fixes this behavior to clear CPU buffers. It also |
341 | * adds support for MSR_IA32_TSX_CTRL which is enumerated by the |
342 | * ARCH_CAP_TSX_CTRL_MSR bit. |
343 | * |
344 | * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode |
345 | * update is required. |
346 | */ |
347 | ia32_cap = x86_read_arch_cap_msr(); |
348 | if ( (ia32_cap & ARCH_CAP_MDS_NO) && |
349 | !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR)) |
350 | taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; |
351 | |
352 | /* |
353 | * TSX is enabled, select alternate mitigation for TAA which is |
354 | * the same as MDS. Enable MDS static branch to clear CPU buffers. |
355 | * |
356 | * For guests that can't determine whether the correct microcode is |
357 | * present on host, enable the mitigation for UCODE_NEEDED as well. |
358 | */ |
359 | static_branch_enable(&mds_user_clear); |
360 | |
361 | if (taa_nosmt || cpu_mitigations_auto_nosmt()) |
362 | cpu_smt_disable(force: false); |
363 | } |
364 | |
365 | static int __init tsx_async_abort_parse_cmdline(char *str) |
366 | { |
367 | if (!boot_cpu_has_bug(X86_BUG_TAA)) |
368 | return 0; |
369 | |
370 | if (!str) |
371 | return -EINVAL; |
372 | |
373 | if (!strcmp(str, "off" )) { |
374 | taa_mitigation = TAA_MITIGATION_OFF; |
375 | } else if (!strcmp(str, "full" )) { |
376 | taa_mitigation = TAA_MITIGATION_VERW; |
377 | } else if (!strcmp(str, "full,nosmt" )) { |
378 | taa_mitigation = TAA_MITIGATION_VERW; |
379 | taa_nosmt = true; |
380 | } |
381 | |
382 | return 0; |
383 | } |
384 | early_param("tsx_async_abort" , tsx_async_abort_parse_cmdline); |
385 | |
386 | #undef pr_fmt |
387 | #define pr_fmt(fmt) "MMIO Stale Data: " fmt |
388 | |
389 | enum mmio_mitigations { |
390 | MMIO_MITIGATION_OFF, |
391 | MMIO_MITIGATION_UCODE_NEEDED, |
392 | MMIO_MITIGATION_VERW, |
393 | }; |
394 | |
395 | /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ |
396 | static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW; |
397 | static bool mmio_nosmt __ro_after_init = false; |
398 | |
399 | static const char * const mmio_strings[] = { |
400 | [MMIO_MITIGATION_OFF] = "Vulnerable" , |
401 | [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode" , |
402 | [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers" , |
403 | }; |
404 | |
405 | static void __init mmio_select_mitigation(void) |
406 | { |
407 | u64 ia32_cap; |
408 | |
409 | if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || |
410 | boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) || |
411 | cpu_mitigations_off()) { |
412 | mmio_mitigation = MMIO_MITIGATION_OFF; |
413 | return; |
414 | } |
415 | |
416 | if (mmio_mitigation == MMIO_MITIGATION_OFF) |
417 | return; |
418 | |
419 | ia32_cap = x86_read_arch_cap_msr(); |
420 | |
421 | /* |
422 | * Enable CPU buffer clear mitigation for host and VMM, if also affected |
423 | * by MDS or TAA. Otherwise, enable mitigation for VMM only. |
424 | */ |
425 | if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) && |
426 | boot_cpu_has(X86_FEATURE_RTM))) |
427 | static_branch_enable(&mds_user_clear); |
428 | else |
429 | static_branch_enable(&mmio_stale_data_clear); |
430 | |
431 | /* |
432 | * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can |
433 | * be propagated to uncore buffers, clearing the Fill buffers on idle |
434 | * is required irrespective of SMT state. |
435 | */ |
436 | if (!(ia32_cap & ARCH_CAP_FBSDP_NO)) |
437 | static_branch_enable(&mds_idle_clear); |
438 | |
439 | /* |
440 | * Check if the system has the right microcode. |
441 | * |
442 | * CPU Fill buffer clear mitigation is enumerated by either an explicit |
443 | * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS |
444 | * affected systems. |
445 | */ |
446 | if ((ia32_cap & ARCH_CAP_FB_CLEAR) || |
447 | (boot_cpu_has(X86_FEATURE_MD_CLEAR) && |
448 | boot_cpu_has(X86_FEATURE_FLUSH_L1D) && |
449 | !(ia32_cap & ARCH_CAP_MDS_NO))) |
450 | mmio_mitigation = MMIO_MITIGATION_VERW; |
451 | else |
452 | mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; |
453 | |
454 | if (mmio_nosmt || cpu_mitigations_auto_nosmt()) |
455 | cpu_smt_disable(force: false); |
456 | } |
457 | |
458 | static int __init mmio_stale_data_parse_cmdline(char *str) |
459 | { |
460 | if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) |
461 | return 0; |
462 | |
463 | if (!str) |
464 | return -EINVAL; |
465 | |
466 | if (!strcmp(str, "off" )) { |
467 | mmio_mitigation = MMIO_MITIGATION_OFF; |
468 | } else if (!strcmp(str, "full" )) { |
469 | mmio_mitigation = MMIO_MITIGATION_VERW; |
470 | } else if (!strcmp(str, "full,nosmt" )) { |
471 | mmio_mitigation = MMIO_MITIGATION_VERW; |
472 | mmio_nosmt = true; |
473 | } |
474 | |
475 | return 0; |
476 | } |
477 | early_param("mmio_stale_data" , mmio_stale_data_parse_cmdline); |
478 | |
479 | #undef pr_fmt |
480 | #define pr_fmt(fmt) "" fmt |
481 | |
482 | static void __init md_clear_update_mitigation(void) |
483 | { |
484 | if (cpu_mitigations_off()) |
485 | return; |
486 | |
487 | if (!static_key_enabled(&mds_user_clear)) |
488 | goto out; |
489 | |
490 | /* |
491 | * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data |
492 | * mitigation, if necessary. |
493 | */ |
494 | if (mds_mitigation == MDS_MITIGATION_OFF && |
495 | boot_cpu_has_bug(X86_BUG_MDS)) { |
496 | mds_mitigation = MDS_MITIGATION_FULL; |
497 | mds_select_mitigation(); |
498 | } |
499 | if (taa_mitigation == TAA_MITIGATION_OFF && |
500 | boot_cpu_has_bug(X86_BUG_TAA)) { |
501 | taa_mitigation = TAA_MITIGATION_VERW; |
502 | taa_select_mitigation(); |
503 | } |
504 | if (mmio_mitigation == MMIO_MITIGATION_OFF && |
505 | boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { |
506 | mmio_mitigation = MMIO_MITIGATION_VERW; |
507 | mmio_select_mitigation(); |
508 | } |
509 | out: |
510 | if (boot_cpu_has_bug(X86_BUG_MDS)) |
511 | pr_info("MDS: %s\n" , mds_strings[mds_mitigation]); |
512 | if (boot_cpu_has_bug(X86_BUG_TAA)) |
513 | pr_info("TAA: %s\n" , taa_strings[taa_mitigation]); |
514 | if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) |
515 | pr_info("MMIO Stale Data: %s\n" , mmio_strings[mmio_mitigation]); |
516 | else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) |
517 | pr_info("MMIO Stale Data: Unknown: No mitigations\n" ); |
518 | } |
519 | |
520 | static void __init md_clear_select_mitigation(void) |
521 | { |
522 | mds_select_mitigation(); |
523 | taa_select_mitigation(); |
524 | mmio_select_mitigation(); |
525 | |
526 | /* |
527 | * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update |
528 | * and print their mitigation after MDS, TAA and MMIO Stale Data |
529 | * mitigation selection is done. |
530 | */ |
531 | md_clear_update_mitigation(); |
532 | } |
533 | |
534 | #undef pr_fmt |
535 | #define pr_fmt(fmt) "SRBDS: " fmt |
536 | |
537 | enum srbds_mitigations { |
538 | SRBDS_MITIGATION_OFF, |
539 | SRBDS_MITIGATION_UCODE_NEEDED, |
540 | SRBDS_MITIGATION_FULL, |
541 | SRBDS_MITIGATION_TSX_OFF, |
542 | SRBDS_MITIGATION_HYPERVISOR, |
543 | }; |
544 | |
545 | static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL; |
546 | |
547 | static const char * const srbds_strings[] = { |
548 | [SRBDS_MITIGATION_OFF] = "Vulnerable" , |
549 | [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode" , |
550 | [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode" , |
551 | [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled" , |
552 | [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status" , |
553 | }; |
554 | |
555 | static bool srbds_off; |
556 | |
557 | void update_srbds_msr(void) |
558 | { |
559 | u64 mcu_ctrl; |
560 | |
561 | if (!boot_cpu_has_bug(X86_BUG_SRBDS)) |
562 | return; |
563 | |
564 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) |
565 | return; |
566 | |
567 | if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) |
568 | return; |
569 | |
570 | /* |
571 | * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX |
572 | * being disabled and it hasn't received the SRBDS MSR microcode. |
573 | */ |
574 | if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) |
575 | return; |
576 | |
577 | rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); |
578 | |
579 | switch (srbds_mitigation) { |
580 | case SRBDS_MITIGATION_OFF: |
581 | case SRBDS_MITIGATION_TSX_OFF: |
582 | mcu_ctrl |= RNGDS_MITG_DIS; |
583 | break; |
584 | case SRBDS_MITIGATION_FULL: |
585 | mcu_ctrl &= ~RNGDS_MITG_DIS; |
586 | break; |
587 | default: |
588 | break; |
589 | } |
590 | |
591 | wrmsrl(MSR_IA32_MCU_OPT_CTRL, val: mcu_ctrl); |
592 | } |
593 | |
594 | static void __init srbds_select_mitigation(void) |
595 | { |
596 | u64 ia32_cap; |
597 | |
598 | if (!boot_cpu_has_bug(X86_BUG_SRBDS)) |
599 | return; |
600 | |
601 | /* |
602 | * Check to see if this is one of the MDS_NO systems supporting TSX that |
603 | * are only exposed to SRBDS when TSX is enabled or when CPU is affected |
604 | * by Processor MMIO Stale Data vulnerability. |
605 | */ |
606 | ia32_cap = x86_read_arch_cap_msr(); |
607 | if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && |
608 | !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) |
609 | srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; |
610 | else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) |
611 | srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; |
612 | else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) |
613 | srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; |
614 | else if (cpu_mitigations_off() || srbds_off) |
615 | srbds_mitigation = SRBDS_MITIGATION_OFF; |
616 | |
617 | update_srbds_msr(); |
618 | pr_info("%s\n" , srbds_strings[srbds_mitigation]); |
619 | } |
620 | |
621 | static int __init srbds_parse_cmdline(char *str) |
622 | { |
623 | if (!str) |
624 | return -EINVAL; |
625 | |
626 | if (!boot_cpu_has_bug(X86_BUG_SRBDS)) |
627 | return 0; |
628 | |
629 | srbds_off = !strcmp(str, "off" ); |
630 | return 0; |
631 | } |
632 | early_param("srbds" , srbds_parse_cmdline); |
633 | |
634 | #undef pr_fmt |
635 | #define pr_fmt(fmt) "L1D Flush : " fmt |
636 | |
637 | enum l1d_flush_mitigations { |
638 | L1D_FLUSH_OFF = 0, |
639 | L1D_FLUSH_ON, |
640 | }; |
641 | |
642 | static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; |
643 | |
644 | static void __init l1d_flush_select_mitigation(void) |
645 | { |
646 | if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) |
647 | return; |
648 | |
649 | static_branch_enable(&switch_mm_cond_l1d_flush); |
650 | pr_info("Conditional flush on switch_mm() enabled\n" ); |
651 | } |
652 | |
653 | static int __init l1d_flush_parse_cmdline(char *str) |
654 | { |
655 | if (!strcmp(str, "on" )) |
656 | l1d_flush_mitigation = L1D_FLUSH_ON; |
657 | |
658 | return 0; |
659 | } |
660 | early_param("l1d_flush" , l1d_flush_parse_cmdline); |
661 | |
662 | #undef pr_fmt |
663 | #define pr_fmt(fmt) "GDS: " fmt |
664 | |
665 | enum gds_mitigations { |
666 | GDS_MITIGATION_OFF, |
667 | GDS_MITIGATION_UCODE_NEEDED, |
668 | GDS_MITIGATION_FORCE, |
669 | GDS_MITIGATION_FULL, |
670 | GDS_MITIGATION_FULL_LOCKED, |
671 | GDS_MITIGATION_HYPERVISOR, |
672 | }; |
673 | |
674 | #if IS_ENABLED(CONFIG_GDS_FORCE_MITIGATION) |
675 | static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FORCE; |
676 | #else |
677 | static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FULL; |
678 | #endif |
679 | |
680 | static const char * const gds_strings[] = { |
681 | [GDS_MITIGATION_OFF] = "Vulnerable" , |
682 | [GDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode" , |
683 | [GDS_MITIGATION_FORCE] = "Mitigation: AVX disabled, no microcode" , |
684 | [GDS_MITIGATION_FULL] = "Mitigation: Microcode" , |
685 | [GDS_MITIGATION_FULL_LOCKED] = "Mitigation: Microcode (locked)" , |
686 | [GDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status" , |
687 | }; |
688 | |
689 | bool gds_ucode_mitigated(void) |
690 | { |
691 | return (gds_mitigation == GDS_MITIGATION_FULL || |
692 | gds_mitigation == GDS_MITIGATION_FULL_LOCKED); |
693 | } |
694 | EXPORT_SYMBOL_GPL(gds_ucode_mitigated); |
695 | |
696 | void update_gds_msr(void) |
697 | { |
698 | u64 mcu_ctrl_after; |
699 | u64 mcu_ctrl; |
700 | |
701 | switch (gds_mitigation) { |
702 | case GDS_MITIGATION_OFF: |
703 | rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); |
704 | mcu_ctrl |= GDS_MITG_DIS; |
705 | break; |
706 | case GDS_MITIGATION_FULL_LOCKED: |
707 | /* |
708 | * The LOCKED state comes from the boot CPU. APs might not have |
709 | * the same state. Make sure the mitigation is enabled on all |
710 | * CPUs. |
711 | */ |
712 | case GDS_MITIGATION_FULL: |
713 | rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); |
714 | mcu_ctrl &= ~GDS_MITG_DIS; |
715 | break; |
716 | case GDS_MITIGATION_FORCE: |
717 | case GDS_MITIGATION_UCODE_NEEDED: |
718 | case GDS_MITIGATION_HYPERVISOR: |
719 | return; |
720 | } |
721 | |
722 | wrmsrl(MSR_IA32_MCU_OPT_CTRL, val: mcu_ctrl); |
723 | |
724 | /* |
725 | * Check to make sure that the WRMSR value was not ignored. Writes to |
726 | * GDS_MITG_DIS will be ignored if this processor is locked but the boot |
727 | * processor was not. |
728 | */ |
729 | rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after); |
730 | WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after); |
731 | } |
732 | |
733 | static void __init gds_select_mitigation(void) |
734 | { |
735 | u64 mcu_ctrl; |
736 | |
737 | if (!boot_cpu_has_bug(X86_BUG_GDS)) |
738 | return; |
739 | |
740 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { |
741 | gds_mitigation = GDS_MITIGATION_HYPERVISOR; |
742 | goto out; |
743 | } |
744 | |
745 | if (cpu_mitigations_off()) |
746 | gds_mitigation = GDS_MITIGATION_OFF; |
747 | /* Will verify below that mitigation _can_ be disabled */ |
748 | |
749 | /* No microcode */ |
750 | if (!(x86_read_arch_cap_msr() & ARCH_CAP_GDS_CTRL)) { |
751 | if (gds_mitigation == GDS_MITIGATION_FORCE) { |
752 | /* |
753 | * This only needs to be done on the boot CPU so do it |
754 | * here rather than in update_gds_msr() |
755 | */ |
756 | setup_clear_cpu_cap(X86_FEATURE_AVX); |
757 | pr_warn("Microcode update needed! Disabling AVX as mitigation.\n" ); |
758 | } else { |
759 | gds_mitigation = GDS_MITIGATION_UCODE_NEEDED; |
760 | } |
761 | goto out; |
762 | } |
763 | |
764 | /* Microcode has mitigation, use it */ |
765 | if (gds_mitigation == GDS_MITIGATION_FORCE) |
766 | gds_mitigation = GDS_MITIGATION_FULL; |
767 | |
768 | rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); |
769 | if (mcu_ctrl & GDS_MITG_LOCKED) { |
770 | if (gds_mitigation == GDS_MITIGATION_OFF) |
771 | pr_warn("Mitigation locked. Disable failed.\n" ); |
772 | |
773 | /* |
774 | * The mitigation is selected from the boot CPU. All other CPUs |
775 | * _should_ have the same state. If the boot CPU isn't locked |
776 | * but others are then update_gds_msr() will WARN() of the state |
777 | * mismatch. If the boot CPU is locked update_gds_msr() will |
778 | * ensure the other CPUs have the mitigation enabled. |
779 | */ |
780 | gds_mitigation = GDS_MITIGATION_FULL_LOCKED; |
781 | } |
782 | |
783 | update_gds_msr(); |
784 | out: |
785 | pr_info("%s\n" , gds_strings[gds_mitigation]); |
786 | } |
787 | |
788 | static int __init gds_parse_cmdline(char *str) |
789 | { |
790 | if (!str) |
791 | return -EINVAL; |
792 | |
793 | if (!boot_cpu_has_bug(X86_BUG_GDS)) |
794 | return 0; |
795 | |
796 | if (!strcmp(str, "off" )) |
797 | gds_mitigation = GDS_MITIGATION_OFF; |
798 | else if (!strcmp(str, "force" )) |
799 | gds_mitigation = GDS_MITIGATION_FORCE; |
800 | |
801 | return 0; |
802 | } |
803 | early_param("gather_data_sampling" , gds_parse_cmdline); |
804 | |
805 | #undef pr_fmt |
806 | #define pr_fmt(fmt) "Spectre V1 : " fmt |
807 | |
808 | enum spectre_v1_mitigation { |
809 | SPECTRE_V1_MITIGATION_NONE, |
810 | SPECTRE_V1_MITIGATION_AUTO, |
811 | }; |
812 | |
813 | static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = |
814 | SPECTRE_V1_MITIGATION_AUTO; |
815 | |
816 | static const char * const spectre_v1_strings[] = { |
817 | [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers" , |
818 | [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization" , |
819 | }; |
820 | |
821 | /* |
822 | * Does SMAP provide full mitigation against speculative kernel access to |
823 | * userspace? |
824 | */ |
825 | static bool smap_works_speculatively(void) |
826 | { |
827 | if (!boot_cpu_has(X86_FEATURE_SMAP)) |
828 | return false; |
829 | |
830 | /* |
831 | * On CPUs which are vulnerable to Meltdown, SMAP does not |
832 | * prevent speculative access to user data in the L1 cache. |
833 | * Consider SMAP to be non-functional as a mitigation on these |
834 | * CPUs. |
835 | */ |
836 | if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) |
837 | return false; |
838 | |
839 | return true; |
840 | } |
841 | |
842 | static void __init spectre_v1_select_mitigation(void) |
843 | { |
844 | if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { |
845 | spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; |
846 | return; |
847 | } |
848 | |
849 | if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { |
850 | /* |
851 | * With Spectre v1, a user can speculatively control either |
852 | * path of a conditional swapgs with a user-controlled GS |
853 | * value. The mitigation is to add lfences to both code paths. |
854 | * |
855 | * If FSGSBASE is enabled, the user can put a kernel address in |
856 | * GS, in which case SMAP provides no protection. |
857 | * |
858 | * If FSGSBASE is disabled, the user can only put a user space |
859 | * address in GS. That makes an attack harder, but still |
860 | * possible if there's no SMAP protection. |
861 | */ |
862 | if (boot_cpu_has(X86_FEATURE_FSGSBASE) || |
863 | !smap_works_speculatively()) { |
864 | /* |
865 | * Mitigation can be provided from SWAPGS itself or |
866 | * PTI as the CR3 write in the Meltdown mitigation |
867 | * is serializing. |
868 | * |
869 | * If neither is there, mitigate with an LFENCE to |
870 | * stop speculation through swapgs. |
871 | */ |
872 | if (boot_cpu_has_bug(X86_BUG_SWAPGS) && |
873 | !boot_cpu_has(X86_FEATURE_PTI)) |
874 | setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); |
875 | |
876 | /* |
877 | * Enable lfences in the kernel entry (non-swapgs) |
878 | * paths, to prevent user entry from speculatively |
879 | * skipping swapgs. |
880 | */ |
881 | setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); |
882 | } |
883 | } |
884 | |
885 | pr_info("%s\n" , spectre_v1_strings[spectre_v1_mitigation]); |
886 | } |
887 | |
888 | static int __init nospectre_v1_cmdline(char *str) |
889 | { |
890 | spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; |
891 | return 0; |
892 | } |
893 | early_param("nospectre_v1" , nospectre_v1_cmdline); |
894 | |
895 | enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; |
896 | |
897 | #undef pr_fmt |
898 | #define pr_fmt(fmt) "RETBleed: " fmt |
899 | |
900 | enum retbleed_mitigation { |
901 | RETBLEED_MITIGATION_NONE, |
902 | RETBLEED_MITIGATION_UNRET, |
903 | RETBLEED_MITIGATION_IBPB, |
904 | RETBLEED_MITIGATION_IBRS, |
905 | RETBLEED_MITIGATION_EIBRS, |
906 | RETBLEED_MITIGATION_STUFF, |
907 | }; |
908 | |
909 | enum retbleed_mitigation_cmd { |
910 | RETBLEED_CMD_OFF, |
911 | RETBLEED_CMD_AUTO, |
912 | RETBLEED_CMD_UNRET, |
913 | RETBLEED_CMD_IBPB, |
914 | RETBLEED_CMD_STUFF, |
915 | }; |
916 | |
917 | static const char * const retbleed_strings[] = { |
918 | [RETBLEED_MITIGATION_NONE] = "Vulnerable" , |
919 | [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk" , |
920 | [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB" , |
921 | [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS" , |
922 | [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS" , |
923 | [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing" , |
924 | }; |
925 | |
926 | static enum retbleed_mitigation retbleed_mitigation __ro_after_init = |
927 | RETBLEED_MITIGATION_NONE; |
928 | static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init = |
929 | RETBLEED_CMD_AUTO; |
930 | |
931 | static int __ro_after_init retbleed_nosmt = false; |
932 | |
933 | static int __init retbleed_parse_cmdline(char *str) |
934 | { |
935 | if (!str) |
936 | return -EINVAL; |
937 | |
938 | while (str) { |
939 | char *next = strchr(str, ','); |
940 | if (next) { |
941 | *next = 0; |
942 | next++; |
943 | } |
944 | |
945 | if (!strcmp(str, "off" )) { |
946 | retbleed_cmd = RETBLEED_CMD_OFF; |
947 | } else if (!strcmp(str, "auto" )) { |
948 | retbleed_cmd = RETBLEED_CMD_AUTO; |
949 | } else if (!strcmp(str, "unret" )) { |
950 | retbleed_cmd = RETBLEED_CMD_UNRET; |
951 | } else if (!strcmp(str, "ibpb" )) { |
952 | retbleed_cmd = RETBLEED_CMD_IBPB; |
953 | } else if (!strcmp(str, "stuff" )) { |
954 | retbleed_cmd = RETBLEED_CMD_STUFF; |
955 | } else if (!strcmp(str, "nosmt" )) { |
956 | retbleed_nosmt = true; |
957 | } else if (!strcmp(str, "force" )) { |
958 | setup_force_cpu_bug(X86_BUG_RETBLEED); |
959 | } else { |
960 | pr_err("Ignoring unknown retbleed option (%s)." , str); |
961 | } |
962 | |
963 | str = next; |
964 | } |
965 | |
966 | return 0; |
967 | } |
968 | early_param("retbleed" , retbleed_parse_cmdline); |
969 | |
970 | #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" |
971 | #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" |
972 | |
973 | static void __init retbleed_select_mitigation(void) |
974 | { |
975 | bool mitigate_smt = false; |
976 | |
977 | if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) |
978 | return; |
979 | |
980 | switch (retbleed_cmd) { |
981 | case RETBLEED_CMD_OFF: |
982 | return; |
983 | |
984 | case RETBLEED_CMD_UNRET: |
985 | if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) { |
986 | retbleed_mitigation = RETBLEED_MITIGATION_UNRET; |
987 | } else { |
988 | pr_err("WARNING: kernel not compiled with CPU_UNRET_ENTRY.\n" ); |
989 | goto do_cmd_auto; |
990 | } |
991 | break; |
992 | |
993 | case RETBLEED_CMD_IBPB: |
994 | if (!boot_cpu_has(X86_FEATURE_IBPB)) { |
995 | pr_err("WARNING: CPU does not support IBPB.\n" ); |
996 | goto do_cmd_auto; |
997 | } else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) { |
998 | retbleed_mitigation = RETBLEED_MITIGATION_IBPB; |
999 | } else { |
1000 | pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n" ); |
1001 | goto do_cmd_auto; |
1002 | } |
1003 | break; |
1004 | |
1005 | case RETBLEED_CMD_STUFF: |
1006 | if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING) && |
1007 | spectre_v2_enabled == SPECTRE_V2_RETPOLINE) { |
1008 | retbleed_mitigation = RETBLEED_MITIGATION_STUFF; |
1009 | |
1010 | } else { |
1011 | if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING)) |
1012 | pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n" ); |
1013 | else |
1014 | pr_err("WARNING: kernel not compiled with CALL_DEPTH_TRACKING.\n" ); |
1015 | |
1016 | goto do_cmd_auto; |
1017 | } |
1018 | break; |
1019 | |
1020 | do_cmd_auto: |
1021 | case RETBLEED_CMD_AUTO: |
1022 | if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || |
1023 | boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { |
1024 | if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) |
1025 | retbleed_mitigation = RETBLEED_MITIGATION_UNRET; |
1026 | else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY) && boot_cpu_has(X86_FEATURE_IBPB)) |
1027 | retbleed_mitigation = RETBLEED_MITIGATION_IBPB; |
1028 | } |
1029 | |
1030 | /* |
1031 | * The Intel mitigation (IBRS or eIBRS) was already selected in |
1032 | * spectre_v2_select_mitigation(). 'retbleed_mitigation' will |
1033 | * be set accordingly below. |
1034 | */ |
1035 | |
1036 | break; |
1037 | } |
1038 | |
1039 | switch (retbleed_mitigation) { |
1040 | case RETBLEED_MITIGATION_UNRET: |
1041 | setup_force_cpu_cap(X86_FEATURE_RETHUNK); |
1042 | setup_force_cpu_cap(X86_FEATURE_UNRET); |
1043 | |
1044 | x86_return_thunk = retbleed_return_thunk; |
1045 | |
1046 | if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && |
1047 | boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) |
1048 | pr_err(RETBLEED_UNTRAIN_MSG); |
1049 | |
1050 | mitigate_smt = true; |
1051 | break; |
1052 | |
1053 | case RETBLEED_MITIGATION_IBPB: |
1054 | setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); |
1055 | setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); |
1056 | mitigate_smt = true; |
1057 | break; |
1058 | |
1059 | case RETBLEED_MITIGATION_STUFF: |
1060 | setup_force_cpu_cap(X86_FEATURE_RETHUNK); |
1061 | setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); |
1062 | |
1063 | x86_return_thunk = call_depth_return_thunk; |
1064 | break; |
1065 | |
1066 | default: |
1067 | break; |
1068 | } |
1069 | |
1070 | if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && |
1071 | (retbleed_nosmt || cpu_mitigations_auto_nosmt())) |
1072 | cpu_smt_disable(force: false); |
1073 | |
1074 | /* |
1075 | * Let IBRS trump all on Intel without affecting the effects of the |
1076 | * retbleed= cmdline option except for call depth based stuffing |
1077 | */ |
1078 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { |
1079 | switch (spectre_v2_enabled) { |
1080 | case SPECTRE_V2_IBRS: |
1081 | retbleed_mitigation = RETBLEED_MITIGATION_IBRS; |
1082 | break; |
1083 | case SPECTRE_V2_EIBRS: |
1084 | case SPECTRE_V2_EIBRS_RETPOLINE: |
1085 | case SPECTRE_V2_EIBRS_LFENCE: |
1086 | retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; |
1087 | break; |
1088 | default: |
1089 | if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) |
1090 | pr_err(RETBLEED_INTEL_MSG); |
1091 | } |
1092 | } |
1093 | |
1094 | pr_info("%s\n" , retbleed_strings[retbleed_mitigation]); |
1095 | } |
1096 | |
1097 | #undef pr_fmt |
1098 | #define pr_fmt(fmt) "Spectre V2 : " fmt |
1099 | |
1100 | static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = |
1101 | SPECTRE_V2_USER_NONE; |
1102 | static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = |
1103 | SPECTRE_V2_USER_NONE; |
1104 | |
1105 | #ifdef CONFIG_RETPOLINE |
1106 | static bool spectre_v2_bad_module; |
1107 | |
1108 | bool retpoline_module_ok(bool has_retpoline) |
1109 | { |
1110 | if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) |
1111 | return true; |
1112 | |
1113 | pr_err("System may be vulnerable to spectre v2\n" ); |
1114 | spectre_v2_bad_module = true; |
1115 | return false; |
1116 | } |
1117 | |
1118 | static inline const char *spectre_v2_module_string(void) |
1119 | { |
1120 | return spectre_v2_bad_module ? " - vulnerable module loaded" : "" ; |
1121 | } |
1122 | #else |
1123 | static inline const char *spectre_v2_module_string(void) { return "" ; } |
1124 | #endif |
1125 | |
1126 | #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" |
1127 | #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" |
1128 | #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" |
1129 | #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" |
1130 | |
1131 | #ifdef CONFIG_BPF_SYSCALL |
1132 | void unpriv_ebpf_notify(int new_state) |
1133 | { |
1134 | if (new_state) |
1135 | return; |
1136 | |
1137 | /* Unprivileged eBPF is enabled */ |
1138 | |
1139 | switch (spectre_v2_enabled) { |
1140 | case SPECTRE_V2_EIBRS: |
1141 | pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); |
1142 | break; |
1143 | case SPECTRE_V2_EIBRS_LFENCE: |
1144 | if (sched_smt_active()) |
1145 | pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); |
1146 | break; |
1147 | default: |
1148 | break; |
1149 | } |
1150 | } |
1151 | #endif |
1152 | |
1153 | static inline bool match_option(const char *arg, int arglen, const char *opt) |
1154 | { |
1155 | int len = strlen(opt); |
1156 | |
1157 | return len == arglen && !strncmp(arg, opt, len); |
1158 | } |
1159 | |
1160 | /* The kernel command line selection for spectre v2 */ |
1161 | enum spectre_v2_mitigation_cmd { |
1162 | SPECTRE_V2_CMD_NONE, |
1163 | SPECTRE_V2_CMD_AUTO, |
1164 | SPECTRE_V2_CMD_FORCE, |
1165 | SPECTRE_V2_CMD_RETPOLINE, |
1166 | SPECTRE_V2_CMD_RETPOLINE_GENERIC, |
1167 | SPECTRE_V2_CMD_RETPOLINE_LFENCE, |
1168 | SPECTRE_V2_CMD_EIBRS, |
1169 | SPECTRE_V2_CMD_EIBRS_RETPOLINE, |
1170 | SPECTRE_V2_CMD_EIBRS_LFENCE, |
1171 | SPECTRE_V2_CMD_IBRS, |
1172 | }; |
1173 | |
1174 | enum spectre_v2_user_cmd { |
1175 | SPECTRE_V2_USER_CMD_NONE, |
1176 | SPECTRE_V2_USER_CMD_AUTO, |
1177 | SPECTRE_V2_USER_CMD_FORCE, |
1178 | SPECTRE_V2_USER_CMD_PRCTL, |
1179 | SPECTRE_V2_USER_CMD_PRCTL_IBPB, |
1180 | SPECTRE_V2_USER_CMD_SECCOMP, |
1181 | SPECTRE_V2_USER_CMD_SECCOMP_IBPB, |
1182 | }; |
1183 | |
1184 | static const char * const spectre_v2_user_strings[] = { |
1185 | [SPECTRE_V2_USER_NONE] = "User space: Vulnerable" , |
1186 | [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection" , |
1187 | [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection" , |
1188 | [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl" , |
1189 | [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl" , |
1190 | }; |
1191 | |
1192 | static const struct { |
1193 | const char *option; |
1194 | enum spectre_v2_user_cmd cmd; |
1195 | bool secure; |
1196 | } v2_user_options[] __initconst = { |
1197 | { "auto" , SPECTRE_V2_USER_CMD_AUTO, false }, |
1198 | { "off" , SPECTRE_V2_USER_CMD_NONE, false }, |
1199 | { "on" , SPECTRE_V2_USER_CMD_FORCE, true }, |
1200 | { "prctl" , SPECTRE_V2_USER_CMD_PRCTL, false }, |
1201 | { "prctl,ibpb" , SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, |
1202 | { "seccomp" , SPECTRE_V2_USER_CMD_SECCOMP, false }, |
1203 | { "seccomp,ibpb" , SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, |
1204 | }; |
1205 | |
1206 | static void __init spec_v2_user_print_cond(const char *reason, bool secure) |
1207 | { |
1208 | if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) |
1209 | pr_info("spectre_v2_user=%s forced on command line.\n" , reason); |
1210 | } |
1211 | |
1212 | static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd; |
1213 | |
1214 | static enum spectre_v2_user_cmd __init |
1215 | spectre_v2_parse_user_cmdline(void) |
1216 | { |
1217 | char arg[20]; |
1218 | int ret, i; |
1219 | |
1220 | switch (spectre_v2_cmd) { |
1221 | case SPECTRE_V2_CMD_NONE: |
1222 | return SPECTRE_V2_USER_CMD_NONE; |
1223 | case SPECTRE_V2_CMD_FORCE: |
1224 | return SPECTRE_V2_USER_CMD_FORCE; |
1225 | default: |
1226 | break; |
1227 | } |
1228 | |
1229 | ret = cmdline_find_option(cmdline_ptr: boot_command_line, option: "spectre_v2_user" , |
1230 | buffer: arg, bufsize: sizeof(arg)); |
1231 | if (ret < 0) |
1232 | return SPECTRE_V2_USER_CMD_AUTO; |
1233 | |
1234 | for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { |
1235 | if (match_option(arg, arglen: ret, opt: v2_user_options[i].option)) { |
1236 | spec_v2_user_print_cond(reason: v2_user_options[i].option, |
1237 | secure: v2_user_options[i].secure); |
1238 | return v2_user_options[i].cmd; |
1239 | } |
1240 | } |
1241 | |
1242 | pr_err("Unknown user space protection option (%s). Switching to AUTO select\n" , arg); |
1243 | return SPECTRE_V2_USER_CMD_AUTO; |
1244 | } |
1245 | |
1246 | static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) |
1247 | { |
1248 | return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; |
1249 | } |
1250 | |
1251 | static void __init |
1252 | spectre_v2_user_select_mitigation(void) |
1253 | { |
1254 | enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; |
1255 | bool smt_possible = IS_ENABLED(CONFIG_SMP); |
1256 | enum spectre_v2_user_cmd cmd; |
1257 | |
1258 | if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) |
1259 | return; |
1260 | |
1261 | if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || |
1262 | cpu_smt_control == CPU_SMT_NOT_SUPPORTED) |
1263 | smt_possible = false; |
1264 | |
1265 | cmd = spectre_v2_parse_user_cmdline(); |
1266 | switch (cmd) { |
1267 | case SPECTRE_V2_USER_CMD_NONE: |
1268 | goto set_mode; |
1269 | case SPECTRE_V2_USER_CMD_FORCE: |
1270 | mode = SPECTRE_V2_USER_STRICT; |
1271 | break; |
1272 | case SPECTRE_V2_USER_CMD_AUTO: |
1273 | case SPECTRE_V2_USER_CMD_PRCTL: |
1274 | case SPECTRE_V2_USER_CMD_PRCTL_IBPB: |
1275 | mode = SPECTRE_V2_USER_PRCTL; |
1276 | break; |
1277 | case SPECTRE_V2_USER_CMD_SECCOMP: |
1278 | case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: |
1279 | if (IS_ENABLED(CONFIG_SECCOMP)) |
1280 | mode = SPECTRE_V2_USER_SECCOMP; |
1281 | else |
1282 | mode = SPECTRE_V2_USER_PRCTL; |
1283 | break; |
1284 | } |
1285 | |
1286 | /* Initialize Indirect Branch Prediction Barrier */ |
1287 | if (boot_cpu_has(X86_FEATURE_IBPB)) { |
1288 | setup_force_cpu_cap(X86_FEATURE_USE_IBPB); |
1289 | |
1290 | spectre_v2_user_ibpb = mode; |
1291 | switch (cmd) { |
1292 | case SPECTRE_V2_USER_CMD_NONE: |
1293 | break; |
1294 | case SPECTRE_V2_USER_CMD_FORCE: |
1295 | case SPECTRE_V2_USER_CMD_PRCTL_IBPB: |
1296 | case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: |
1297 | static_branch_enable(&switch_mm_always_ibpb); |
1298 | spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; |
1299 | break; |
1300 | case SPECTRE_V2_USER_CMD_PRCTL: |
1301 | case SPECTRE_V2_USER_CMD_AUTO: |
1302 | case SPECTRE_V2_USER_CMD_SECCOMP: |
1303 | static_branch_enable(&switch_mm_cond_ibpb); |
1304 | break; |
1305 | } |
1306 | |
1307 | pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n" , |
1308 | static_key_enabled(&switch_mm_always_ibpb) ? |
1309 | "always-on" : "conditional" ); |
1310 | } |
1311 | |
1312 | /* |
1313 | * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP |
1314 | * is not required. |
1315 | * |
1316 | * Intel's Enhanced IBRS also protects against cross-thread branch target |
1317 | * injection in user-mode as the IBRS bit remains always set which |
1318 | * implicitly enables cross-thread protections. However, in legacy IBRS |
1319 | * mode, the IBRS bit is set only on kernel entry and cleared on return |
1320 | * to userspace. AMD Automatic IBRS also does not protect userspace. |
1321 | * These modes therefore disable the implicit cross-thread protection, |
1322 | * so allow for STIBP to be selected in those cases. |
1323 | */ |
1324 | if (!boot_cpu_has(X86_FEATURE_STIBP) || |
1325 | !smt_possible || |
1326 | (spectre_v2_in_eibrs_mode(mode: spectre_v2_enabled) && |
1327 | !boot_cpu_has(X86_FEATURE_AUTOIBRS))) |
1328 | return; |
1329 | |
1330 | /* |
1331 | * At this point, an STIBP mode other than "off" has been set. |
1332 | * If STIBP support is not being forced, check if STIBP always-on |
1333 | * is preferred. |
1334 | */ |
1335 | if (mode != SPECTRE_V2_USER_STRICT && |
1336 | boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) |
1337 | mode = SPECTRE_V2_USER_STRICT_PREFERRED; |
1338 | |
1339 | if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || |
1340 | retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { |
1341 | if (mode != SPECTRE_V2_USER_STRICT && |
1342 | mode != SPECTRE_V2_USER_STRICT_PREFERRED) |
1343 | pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n" ); |
1344 | mode = SPECTRE_V2_USER_STRICT_PREFERRED; |
1345 | } |
1346 | |
1347 | spectre_v2_user_stibp = mode; |
1348 | |
1349 | set_mode: |
1350 | pr_info("%s\n" , spectre_v2_user_strings[mode]); |
1351 | } |
1352 | |
1353 | static const char * const spectre_v2_strings[] = { |
1354 | [SPECTRE_V2_NONE] = "Vulnerable" , |
1355 | [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines" , |
1356 | [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE" , |
1357 | [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS" , |
1358 | [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE" , |
1359 | [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines" , |
1360 | [SPECTRE_V2_IBRS] = "Mitigation: IBRS" , |
1361 | }; |
1362 | |
1363 | static const struct { |
1364 | const char *option; |
1365 | enum spectre_v2_mitigation_cmd cmd; |
1366 | bool secure; |
1367 | } mitigation_options[] __initconst = { |
1368 | { "off" , SPECTRE_V2_CMD_NONE, false }, |
1369 | { "on" , SPECTRE_V2_CMD_FORCE, true }, |
1370 | { "retpoline" , SPECTRE_V2_CMD_RETPOLINE, false }, |
1371 | { "retpoline,amd" , SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, |
1372 | { "retpoline,lfence" , SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, |
1373 | { "retpoline,generic" , SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, |
1374 | { "eibrs" , SPECTRE_V2_CMD_EIBRS, false }, |
1375 | { "eibrs,lfence" , SPECTRE_V2_CMD_EIBRS_LFENCE, false }, |
1376 | { "eibrs,retpoline" , SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, |
1377 | { "auto" , SPECTRE_V2_CMD_AUTO, false }, |
1378 | { "ibrs" , SPECTRE_V2_CMD_IBRS, false }, |
1379 | }; |
1380 | |
1381 | static void __init spec_v2_print_cond(const char *reason, bool secure) |
1382 | { |
1383 | if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) |
1384 | pr_info("%s selected on command line.\n" , reason); |
1385 | } |
1386 | |
1387 | static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) |
1388 | { |
1389 | enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; |
1390 | char arg[20]; |
1391 | int ret, i; |
1392 | |
1393 | if (cmdline_find_option_bool(cmdline_ptr: boot_command_line, option: "nospectre_v2" ) || |
1394 | cpu_mitigations_off()) |
1395 | return SPECTRE_V2_CMD_NONE; |
1396 | |
1397 | ret = cmdline_find_option(cmdline_ptr: boot_command_line, option: "spectre_v2" , buffer: arg, bufsize: sizeof(arg)); |
1398 | if (ret < 0) |
1399 | return SPECTRE_V2_CMD_AUTO; |
1400 | |
1401 | for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { |
1402 | if (!match_option(arg, arglen: ret, opt: mitigation_options[i].option)) |
1403 | continue; |
1404 | cmd = mitigation_options[i].cmd; |
1405 | break; |
1406 | } |
1407 | |
1408 | if (i >= ARRAY_SIZE(mitigation_options)) { |
1409 | pr_err("unknown option (%s). Switching to AUTO select\n" , arg); |
1410 | return SPECTRE_V2_CMD_AUTO; |
1411 | } |
1412 | |
1413 | if ((cmd == SPECTRE_V2_CMD_RETPOLINE || |
1414 | cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || |
1415 | cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || |
1416 | cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || |
1417 | cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && |
1418 | !IS_ENABLED(CONFIG_RETPOLINE)) { |
1419 | pr_err("%s selected but not compiled in. Switching to AUTO select\n" , |
1420 | mitigation_options[i].option); |
1421 | return SPECTRE_V2_CMD_AUTO; |
1422 | } |
1423 | |
1424 | if ((cmd == SPECTRE_V2_CMD_EIBRS || |
1425 | cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || |
1426 | cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && |
1427 | !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { |
1428 | pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n" , |
1429 | mitigation_options[i].option); |
1430 | return SPECTRE_V2_CMD_AUTO; |
1431 | } |
1432 | |
1433 | if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || |
1434 | cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && |
1435 | !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { |
1436 | pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n" , |
1437 | mitigation_options[i].option); |
1438 | return SPECTRE_V2_CMD_AUTO; |
1439 | } |
1440 | |
1441 | if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_CPU_IBRS_ENTRY)) { |
1442 | pr_err("%s selected but not compiled in. Switching to AUTO select\n" , |
1443 | mitigation_options[i].option); |
1444 | return SPECTRE_V2_CMD_AUTO; |
1445 | } |
1446 | |
1447 | if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { |
1448 | pr_err("%s selected but not Intel CPU. Switching to AUTO select\n" , |
1449 | mitigation_options[i].option); |
1450 | return SPECTRE_V2_CMD_AUTO; |
1451 | } |
1452 | |
1453 | if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { |
1454 | pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n" , |
1455 | mitigation_options[i].option); |
1456 | return SPECTRE_V2_CMD_AUTO; |
1457 | } |
1458 | |
1459 | if (cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { |
1460 | pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n" , |
1461 | mitigation_options[i].option); |
1462 | return SPECTRE_V2_CMD_AUTO; |
1463 | } |
1464 | |
1465 | spec_v2_print_cond(reason: mitigation_options[i].option, |
1466 | secure: mitigation_options[i].secure); |
1467 | return cmd; |
1468 | } |
1469 | |
1470 | static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) |
1471 | { |
1472 | if (!IS_ENABLED(CONFIG_RETPOLINE)) { |
1473 | pr_err("Kernel not compiled with retpoline; no mitigation available!" ); |
1474 | return SPECTRE_V2_NONE; |
1475 | } |
1476 | |
1477 | return SPECTRE_V2_RETPOLINE; |
1478 | } |
1479 | |
1480 | /* Disable in-kernel use of non-RSB RET predictors */ |
1481 | static void __init spec_ctrl_disable_kernel_rrsba(void) |
1482 | { |
1483 | u64 ia32_cap; |
1484 | |
1485 | if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) |
1486 | return; |
1487 | |
1488 | ia32_cap = x86_read_arch_cap_msr(); |
1489 | |
1490 | if (ia32_cap & ARCH_CAP_RRSBA) { |
1491 | x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; |
1492 | update_spec_ctrl(val: x86_spec_ctrl_base); |
1493 | } |
1494 | } |
1495 | |
1496 | static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode) |
1497 | { |
1498 | /* |
1499 | * Similar to context switches, there are two types of RSB attacks |
1500 | * after VM exit: |
1501 | * |
1502 | * 1) RSB underflow |
1503 | * |
1504 | * 2) Poisoned RSB entry |
1505 | * |
1506 | * When retpoline is enabled, both are mitigated by filling/clearing |
1507 | * the RSB. |
1508 | * |
1509 | * When IBRS is enabled, while #1 would be mitigated by the IBRS branch |
1510 | * prediction isolation protections, RSB still needs to be cleared |
1511 | * because of #2. Note that SMEP provides no protection here, unlike |
1512 | * user-space-poisoned RSB entries. |
1513 | * |
1514 | * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB |
1515 | * bug is present then a LITE version of RSB protection is required, |
1516 | * just a single call needs to retire before a RET is executed. |
1517 | */ |
1518 | switch (mode) { |
1519 | case SPECTRE_V2_NONE: |
1520 | return; |
1521 | |
1522 | case SPECTRE_V2_EIBRS_LFENCE: |
1523 | case SPECTRE_V2_EIBRS: |
1524 | if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { |
1525 | setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); |
1526 | pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n" ); |
1527 | } |
1528 | return; |
1529 | |
1530 | case SPECTRE_V2_EIBRS_RETPOLINE: |
1531 | case SPECTRE_V2_RETPOLINE: |
1532 | case SPECTRE_V2_LFENCE: |
1533 | case SPECTRE_V2_IBRS: |
1534 | setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); |
1535 | pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n" ); |
1536 | return; |
1537 | } |
1538 | |
1539 | pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit" ); |
1540 | dump_stack(); |
1541 | } |
1542 | |
1543 | static void __init spectre_v2_select_mitigation(void) |
1544 | { |
1545 | enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); |
1546 | enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; |
1547 | |
1548 | /* |
1549 | * If the CPU is not affected and the command line mode is NONE or AUTO |
1550 | * then nothing to do. |
1551 | */ |
1552 | if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && |
1553 | (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) |
1554 | return; |
1555 | |
1556 | switch (cmd) { |
1557 | case SPECTRE_V2_CMD_NONE: |
1558 | return; |
1559 | |
1560 | case SPECTRE_V2_CMD_FORCE: |
1561 | case SPECTRE_V2_CMD_AUTO: |
1562 | if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { |
1563 | mode = SPECTRE_V2_EIBRS; |
1564 | break; |
1565 | } |
1566 | |
1567 | if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) && |
1568 | boot_cpu_has_bug(X86_BUG_RETBLEED) && |
1569 | retbleed_cmd != RETBLEED_CMD_OFF && |
1570 | retbleed_cmd != RETBLEED_CMD_STUFF && |
1571 | boot_cpu_has(X86_FEATURE_IBRS) && |
1572 | boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { |
1573 | mode = SPECTRE_V2_IBRS; |
1574 | break; |
1575 | } |
1576 | |
1577 | mode = spectre_v2_select_retpoline(); |
1578 | break; |
1579 | |
1580 | case SPECTRE_V2_CMD_RETPOLINE_LFENCE: |
1581 | pr_err(SPECTRE_V2_LFENCE_MSG); |
1582 | mode = SPECTRE_V2_LFENCE; |
1583 | break; |
1584 | |
1585 | case SPECTRE_V2_CMD_RETPOLINE_GENERIC: |
1586 | mode = SPECTRE_V2_RETPOLINE; |
1587 | break; |
1588 | |
1589 | case SPECTRE_V2_CMD_RETPOLINE: |
1590 | mode = spectre_v2_select_retpoline(); |
1591 | break; |
1592 | |
1593 | case SPECTRE_V2_CMD_IBRS: |
1594 | mode = SPECTRE_V2_IBRS; |
1595 | break; |
1596 | |
1597 | case SPECTRE_V2_CMD_EIBRS: |
1598 | mode = SPECTRE_V2_EIBRS; |
1599 | break; |
1600 | |
1601 | case SPECTRE_V2_CMD_EIBRS_LFENCE: |
1602 | mode = SPECTRE_V2_EIBRS_LFENCE; |
1603 | break; |
1604 | |
1605 | case SPECTRE_V2_CMD_EIBRS_RETPOLINE: |
1606 | mode = SPECTRE_V2_EIBRS_RETPOLINE; |
1607 | break; |
1608 | } |
1609 | |
1610 | if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) |
1611 | pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); |
1612 | |
1613 | if (spectre_v2_in_ibrs_mode(mode)) { |
1614 | if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { |
1615 | msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); |
1616 | } else { |
1617 | x86_spec_ctrl_base |= SPEC_CTRL_IBRS; |
1618 | update_spec_ctrl(val: x86_spec_ctrl_base); |
1619 | } |
1620 | } |
1621 | |
1622 | switch (mode) { |
1623 | case SPECTRE_V2_NONE: |
1624 | case SPECTRE_V2_EIBRS: |
1625 | break; |
1626 | |
1627 | case SPECTRE_V2_IBRS: |
1628 | setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); |
1629 | if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) |
1630 | pr_warn(SPECTRE_V2_IBRS_PERF_MSG); |
1631 | break; |
1632 | |
1633 | case SPECTRE_V2_LFENCE: |
1634 | case SPECTRE_V2_EIBRS_LFENCE: |
1635 | setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); |
1636 | fallthrough; |
1637 | |
1638 | case SPECTRE_V2_RETPOLINE: |
1639 | case SPECTRE_V2_EIBRS_RETPOLINE: |
1640 | setup_force_cpu_cap(X86_FEATURE_RETPOLINE); |
1641 | break; |
1642 | } |
1643 | |
1644 | /* |
1645 | * Disable alternate RSB predictions in kernel when indirect CALLs and |
1646 | * JMPs gets protection against BHI and Intramode-BTI, but RET |
1647 | * prediction from a non-RSB predictor is still a risk. |
1648 | */ |
1649 | if (mode == SPECTRE_V2_EIBRS_LFENCE || |
1650 | mode == SPECTRE_V2_EIBRS_RETPOLINE || |
1651 | mode == SPECTRE_V2_RETPOLINE) |
1652 | spec_ctrl_disable_kernel_rrsba(); |
1653 | |
1654 | spectre_v2_enabled = mode; |
1655 | pr_info("%s\n" , spectre_v2_strings[mode]); |
1656 | |
1657 | /* |
1658 | * If Spectre v2 protection has been enabled, fill the RSB during a |
1659 | * context switch. In general there are two types of RSB attacks |
1660 | * across context switches, for which the CALLs/RETs may be unbalanced. |
1661 | * |
1662 | * 1) RSB underflow |
1663 | * |
1664 | * Some Intel parts have "bottomless RSB". When the RSB is empty, |
1665 | * speculated return targets may come from the branch predictor, |
1666 | * which could have a user-poisoned BTB or BHB entry. |
1667 | * |
1668 | * AMD has it even worse: *all* returns are speculated from the BTB, |
1669 | * regardless of the state of the RSB. |
1670 | * |
1671 | * When IBRS or eIBRS is enabled, the "user -> kernel" attack |
1672 | * scenario is mitigated by the IBRS branch prediction isolation |
1673 | * properties, so the RSB buffer filling wouldn't be necessary to |
1674 | * protect against this type of attack. |
1675 | * |
1676 | * The "user -> user" attack scenario is mitigated by RSB filling. |
1677 | * |
1678 | * 2) Poisoned RSB entry |
1679 | * |
1680 | * If the 'next' in-kernel return stack is shorter than 'prev', |
1681 | * 'next' could be tricked into speculating with a user-poisoned RSB |
1682 | * entry. |
1683 | * |
1684 | * The "user -> kernel" attack scenario is mitigated by SMEP and |
1685 | * eIBRS. |
1686 | * |
1687 | * The "user -> user" scenario, also known as SpectreBHB, requires |
1688 | * RSB clearing. |
1689 | * |
1690 | * So to mitigate all cases, unconditionally fill RSB on context |
1691 | * switches. |
1692 | * |
1693 | * FIXME: Is this pointless for retbleed-affected AMD? |
1694 | */ |
1695 | setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); |
1696 | pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n" ); |
1697 | |
1698 | spectre_v2_determine_rsb_fill_type_at_vmexit(mode); |
1699 | |
1700 | /* |
1701 | * Retpoline protects the kernel, but doesn't protect firmware. IBRS |
1702 | * and Enhanced IBRS protect firmware too, so enable IBRS around |
1703 | * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't |
1704 | * otherwise enabled. |
1705 | * |
1706 | * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because |
1707 | * the user might select retpoline on the kernel command line and if |
1708 | * the CPU supports Enhanced IBRS, kernel might un-intentionally not |
1709 | * enable IBRS around firmware calls. |
1710 | */ |
1711 | if (boot_cpu_has_bug(X86_BUG_RETBLEED) && |
1712 | boot_cpu_has(X86_FEATURE_IBPB) && |
1713 | (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || |
1714 | boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { |
1715 | |
1716 | if (retbleed_cmd != RETBLEED_CMD_IBPB) { |
1717 | setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); |
1718 | pr_info("Enabling Speculation Barrier for firmware calls\n" ); |
1719 | } |
1720 | |
1721 | } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) { |
1722 | setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); |
1723 | pr_info("Enabling Restricted Speculation for firmware calls\n" ); |
1724 | } |
1725 | |
1726 | /* Set up IBPB and STIBP depending on the general spectre V2 command */ |
1727 | spectre_v2_cmd = cmd; |
1728 | } |
1729 | |
1730 | static void update_stibp_msr(void * __unused) |
1731 | { |
1732 | u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); |
1733 | update_spec_ctrl(val); |
1734 | } |
1735 | |
1736 | /* Update x86_spec_ctrl_base in case SMT state changed. */ |
1737 | static void update_stibp_strict(void) |
1738 | { |
1739 | u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; |
1740 | |
1741 | if (sched_smt_active()) |
1742 | mask |= SPEC_CTRL_STIBP; |
1743 | |
1744 | if (mask == x86_spec_ctrl_base) |
1745 | return; |
1746 | |
1747 | pr_info("Update user space SMT mitigation: STIBP %s\n" , |
1748 | mask & SPEC_CTRL_STIBP ? "always-on" : "off" ); |
1749 | x86_spec_ctrl_base = mask; |
1750 | on_each_cpu(func: update_stibp_msr, NULL, wait: 1); |
1751 | } |
1752 | |
1753 | /* Update the static key controlling the evaluation of TIF_SPEC_IB */ |
1754 | static void update_indir_branch_cond(void) |
1755 | { |
1756 | if (sched_smt_active()) |
1757 | static_branch_enable(&switch_to_cond_stibp); |
1758 | else |
1759 | static_branch_disable(&switch_to_cond_stibp); |
1760 | } |
1761 | |
1762 | #undef pr_fmt |
1763 | #define pr_fmt(fmt) fmt |
1764 | |
1765 | /* Update the static key controlling the MDS CPU buffer clear in idle */ |
1766 | static void update_mds_branch_idle(void) |
1767 | { |
1768 | u64 ia32_cap = x86_read_arch_cap_msr(); |
1769 | |
1770 | /* |
1771 | * Enable the idle clearing if SMT is active on CPUs which are |
1772 | * affected only by MSBDS and not any other MDS variant. |
1773 | * |
1774 | * The other variants cannot be mitigated when SMT is enabled, so |
1775 | * clearing the buffers on idle just to prevent the Store Buffer |
1776 | * repartitioning leak would be a window dressing exercise. |
1777 | */ |
1778 | if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) |
1779 | return; |
1780 | |
1781 | if (sched_smt_active()) { |
1782 | static_branch_enable(&mds_idle_clear); |
1783 | } else if (mmio_mitigation == MMIO_MITIGATION_OFF || |
1784 | (ia32_cap & ARCH_CAP_FBSDP_NO)) { |
1785 | static_branch_disable(&mds_idle_clear); |
1786 | } |
1787 | } |
1788 | |
1789 | #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" |
1790 | #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" |
1791 | #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" |
1792 | |
1793 | void cpu_bugs_smt_update(void) |
1794 | { |
1795 | mutex_lock(&spec_ctrl_mutex); |
1796 | |
1797 | if (sched_smt_active() && unprivileged_ebpf_enabled() && |
1798 | spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) |
1799 | pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); |
1800 | |
1801 | switch (spectre_v2_user_stibp) { |
1802 | case SPECTRE_V2_USER_NONE: |
1803 | break; |
1804 | case SPECTRE_V2_USER_STRICT: |
1805 | case SPECTRE_V2_USER_STRICT_PREFERRED: |
1806 | update_stibp_strict(); |
1807 | break; |
1808 | case SPECTRE_V2_USER_PRCTL: |
1809 | case SPECTRE_V2_USER_SECCOMP: |
1810 | update_indir_branch_cond(); |
1811 | break; |
1812 | } |
1813 | |
1814 | switch (mds_mitigation) { |
1815 | case MDS_MITIGATION_FULL: |
1816 | case MDS_MITIGATION_VMWERV: |
1817 | if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) |
1818 | pr_warn_once(MDS_MSG_SMT); |
1819 | update_mds_branch_idle(); |
1820 | break; |
1821 | case MDS_MITIGATION_OFF: |
1822 | break; |
1823 | } |
1824 | |
1825 | switch (taa_mitigation) { |
1826 | case TAA_MITIGATION_VERW: |
1827 | case TAA_MITIGATION_UCODE_NEEDED: |
1828 | if (sched_smt_active()) |
1829 | pr_warn_once(TAA_MSG_SMT); |
1830 | break; |
1831 | case TAA_MITIGATION_TSX_DISABLED: |
1832 | case TAA_MITIGATION_OFF: |
1833 | break; |
1834 | } |
1835 | |
1836 | switch (mmio_mitigation) { |
1837 | case MMIO_MITIGATION_VERW: |
1838 | case MMIO_MITIGATION_UCODE_NEEDED: |
1839 | if (sched_smt_active()) |
1840 | pr_warn_once(MMIO_MSG_SMT); |
1841 | break; |
1842 | case MMIO_MITIGATION_OFF: |
1843 | break; |
1844 | } |
1845 | |
1846 | mutex_unlock(lock: &spec_ctrl_mutex); |
1847 | } |
1848 | |
1849 | #undef pr_fmt |
1850 | #define pr_fmt(fmt) "Speculative Store Bypass: " fmt |
1851 | |
1852 | static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; |
1853 | |
1854 | /* The kernel command line selection */ |
1855 | enum ssb_mitigation_cmd { |
1856 | SPEC_STORE_BYPASS_CMD_NONE, |
1857 | SPEC_STORE_BYPASS_CMD_AUTO, |
1858 | SPEC_STORE_BYPASS_CMD_ON, |
1859 | SPEC_STORE_BYPASS_CMD_PRCTL, |
1860 | SPEC_STORE_BYPASS_CMD_SECCOMP, |
1861 | }; |
1862 | |
1863 | static const char * const ssb_strings[] = { |
1864 | [SPEC_STORE_BYPASS_NONE] = "Vulnerable" , |
1865 | [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled" , |
1866 | [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl" , |
1867 | [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp" , |
1868 | }; |
1869 | |
1870 | static const struct { |
1871 | const char *option; |
1872 | enum ssb_mitigation_cmd cmd; |
1873 | } ssb_mitigation_options[] __initconst = { |
1874 | { "auto" , SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ |
1875 | { "on" , SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ |
1876 | { "off" , SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ |
1877 | { "prctl" , SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ |
1878 | { "seccomp" , SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ |
1879 | }; |
1880 | |
1881 | static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) |
1882 | { |
1883 | enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO; |
1884 | char arg[20]; |
1885 | int ret, i; |
1886 | |
1887 | if (cmdline_find_option_bool(cmdline_ptr: boot_command_line, option: "nospec_store_bypass_disable" ) || |
1888 | cpu_mitigations_off()) { |
1889 | return SPEC_STORE_BYPASS_CMD_NONE; |
1890 | } else { |
1891 | ret = cmdline_find_option(cmdline_ptr: boot_command_line, option: "spec_store_bypass_disable" , |
1892 | buffer: arg, bufsize: sizeof(arg)); |
1893 | if (ret < 0) |
1894 | return SPEC_STORE_BYPASS_CMD_AUTO; |
1895 | |
1896 | for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { |
1897 | if (!match_option(arg, arglen: ret, opt: ssb_mitigation_options[i].option)) |
1898 | continue; |
1899 | |
1900 | cmd = ssb_mitigation_options[i].cmd; |
1901 | break; |
1902 | } |
1903 | |
1904 | if (i >= ARRAY_SIZE(ssb_mitigation_options)) { |
1905 | pr_err("unknown option (%s). Switching to AUTO select\n" , arg); |
1906 | return SPEC_STORE_BYPASS_CMD_AUTO; |
1907 | } |
1908 | } |
1909 | |
1910 | return cmd; |
1911 | } |
1912 | |
1913 | static enum ssb_mitigation __init __ssb_select_mitigation(void) |
1914 | { |
1915 | enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; |
1916 | enum ssb_mitigation_cmd cmd; |
1917 | |
1918 | if (!boot_cpu_has(X86_FEATURE_SSBD)) |
1919 | return mode; |
1920 | |
1921 | cmd = ssb_parse_cmdline(); |
1922 | if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && |
1923 | (cmd == SPEC_STORE_BYPASS_CMD_NONE || |
1924 | cmd == SPEC_STORE_BYPASS_CMD_AUTO)) |
1925 | return mode; |
1926 | |
1927 | switch (cmd) { |
1928 | case SPEC_STORE_BYPASS_CMD_SECCOMP: |
1929 | /* |
1930 | * Choose prctl+seccomp as the default mode if seccomp is |
1931 | * enabled. |
1932 | */ |
1933 | if (IS_ENABLED(CONFIG_SECCOMP)) |
1934 | mode = SPEC_STORE_BYPASS_SECCOMP; |
1935 | else |
1936 | mode = SPEC_STORE_BYPASS_PRCTL; |
1937 | break; |
1938 | case SPEC_STORE_BYPASS_CMD_ON: |
1939 | mode = SPEC_STORE_BYPASS_DISABLE; |
1940 | break; |
1941 | case SPEC_STORE_BYPASS_CMD_AUTO: |
1942 | case SPEC_STORE_BYPASS_CMD_PRCTL: |
1943 | mode = SPEC_STORE_BYPASS_PRCTL; |
1944 | break; |
1945 | case SPEC_STORE_BYPASS_CMD_NONE: |
1946 | break; |
1947 | } |
1948 | |
1949 | /* |
1950 | * We have three CPU feature flags that are in play here: |
1951 | * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. |
1952 | * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass |
1953 | * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation |
1954 | */ |
1955 | if (mode == SPEC_STORE_BYPASS_DISABLE) { |
1956 | setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); |
1957 | /* |
1958 | * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may |
1959 | * use a completely different MSR and bit dependent on family. |
1960 | */ |
1961 | if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && |
1962 | !static_cpu_has(X86_FEATURE_AMD_SSBD)) { |
1963 | x86_amd_ssb_disable(); |
1964 | } else { |
1965 | x86_spec_ctrl_base |= SPEC_CTRL_SSBD; |
1966 | update_spec_ctrl(val: x86_spec_ctrl_base); |
1967 | } |
1968 | } |
1969 | |
1970 | return mode; |
1971 | } |
1972 | |
1973 | static void ssb_select_mitigation(void) |
1974 | { |
1975 | ssb_mode = __ssb_select_mitigation(); |
1976 | |
1977 | if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) |
1978 | pr_info("%s\n" , ssb_strings[ssb_mode]); |
1979 | } |
1980 | |
1981 | #undef pr_fmt |
1982 | #define pr_fmt(fmt) "Speculation prctl: " fmt |
1983 | |
1984 | static void task_update_spec_tif(struct task_struct *tsk) |
1985 | { |
1986 | /* Force the update of the real TIF bits */ |
1987 | set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); |
1988 | |
1989 | /* |
1990 | * Immediately update the speculation control MSRs for the current |
1991 | * task, but for a non-current task delay setting the CPU |
1992 | * mitigation until it is scheduled next. |
1993 | * |
1994 | * This can only happen for SECCOMP mitigation. For PRCTL it's |
1995 | * always the current task. |
1996 | */ |
1997 | if (tsk == current) |
1998 | speculation_ctrl_update_current(); |
1999 | } |
2000 | |
2001 | static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) |
2002 | { |
2003 | |
2004 | if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) |
2005 | return -EPERM; |
2006 | |
2007 | switch (ctrl) { |
2008 | case PR_SPEC_ENABLE: |
2009 | set_ti_thread_flag(ti: &task->thread_info, TIF_SPEC_L1D_FLUSH); |
2010 | return 0; |
2011 | case PR_SPEC_DISABLE: |
2012 | clear_ti_thread_flag(ti: &task->thread_info, TIF_SPEC_L1D_FLUSH); |
2013 | return 0; |
2014 | default: |
2015 | return -ERANGE; |
2016 | } |
2017 | } |
2018 | |
2019 | static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) |
2020 | { |
2021 | if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && |
2022 | ssb_mode != SPEC_STORE_BYPASS_SECCOMP) |
2023 | return -ENXIO; |
2024 | |
2025 | switch (ctrl) { |
2026 | case PR_SPEC_ENABLE: |
2027 | /* If speculation is force disabled, enable is not allowed */ |
2028 | if (task_spec_ssb_force_disable(p: task)) |
2029 | return -EPERM; |
2030 | task_clear_spec_ssb_disable(p: task); |
2031 | task_clear_spec_ssb_noexec(p: task); |
2032 | task_update_spec_tif(tsk: task); |
2033 | break; |
2034 | case PR_SPEC_DISABLE: |
2035 | task_set_spec_ssb_disable(p: task); |
2036 | task_clear_spec_ssb_noexec(p: task); |
2037 | task_update_spec_tif(tsk: task); |
2038 | break; |
2039 | case PR_SPEC_FORCE_DISABLE: |
2040 | task_set_spec_ssb_disable(p: task); |
2041 | task_set_spec_ssb_force_disable(p: task); |
2042 | task_clear_spec_ssb_noexec(p: task); |
2043 | task_update_spec_tif(tsk: task); |
2044 | break; |
2045 | case PR_SPEC_DISABLE_NOEXEC: |
2046 | if (task_spec_ssb_force_disable(p: task)) |
2047 | return -EPERM; |
2048 | task_set_spec_ssb_disable(p: task); |
2049 | task_set_spec_ssb_noexec(p: task); |
2050 | task_update_spec_tif(tsk: task); |
2051 | break; |
2052 | default: |
2053 | return -ERANGE; |
2054 | } |
2055 | return 0; |
2056 | } |
2057 | |
2058 | static bool is_spec_ib_user_controlled(void) |
2059 | { |
2060 | return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || |
2061 | spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || |
2062 | spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || |
2063 | spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; |
2064 | } |
2065 | |
2066 | static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) |
2067 | { |
2068 | switch (ctrl) { |
2069 | case PR_SPEC_ENABLE: |
2070 | if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && |
2071 | spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) |
2072 | return 0; |
2073 | |
2074 | /* |
2075 | * With strict mode for both IBPB and STIBP, the instruction |
2076 | * code paths avoid checking this task flag and instead, |
2077 | * unconditionally run the instruction. However, STIBP and IBPB |
2078 | * are independent and either can be set to conditionally |
2079 | * enabled regardless of the mode of the other. |
2080 | * |
2081 | * If either is set to conditional, allow the task flag to be |
2082 | * updated, unless it was force-disabled by a previous prctl |
2083 | * call. Currently, this is possible on an AMD CPU which has the |
2084 | * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the |
2085 | * kernel is booted with 'spectre_v2_user=seccomp', then |
2086 | * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and |
2087 | * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. |
2088 | */ |
2089 | if (!is_spec_ib_user_controlled() || |
2090 | task_spec_ib_force_disable(p: task)) |
2091 | return -EPERM; |
2092 | |
2093 | task_clear_spec_ib_disable(p: task); |
2094 | task_update_spec_tif(tsk: task); |
2095 | break; |
2096 | case PR_SPEC_DISABLE: |
2097 | case PR_SPEC_FORCE_DISABLE: |
2098 | /* |
2099 | * Indirect branch speculation is always allowed when |
2100 | * mitigation is force disabled. |
2101 | */ |
2102 | if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && |
2103 | spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) |
2104 | return -EPERM; |
2105 | |
2106 | if (!is_spec_ib_user_controlled()) |
2107 | return 0; |
2108 | |
2109 | task_set_spec_ib_disable(p: task); |
2110 | if (ctrl == PR_SPEC_FORCE_DISABLE) |
2111 | task_set_spec_ib_force_disable(p: task); |
2112 | task_update_spec_tif(tsk: task); |
2113 | if (task == current) |
2114 | indirect_branch_prediction_barrier(); |
2115 | break; |
2116 | default: |
2117 | return -ERANGE; |
2118 | } |
2119 | return 0; |
2120 | } |
2121 | |
2122 | int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, |
2123 | unsigned long ctrl) |
2124 | { |
2125 | switch (which) { |
2126 | case PR_SPEC_STORE_BYPASS: |
2127 | return ssb_prctl_set(task, ctrl); |
2128 | case PR_SPEC_INDIRECT_BRANCH: |
2129 | return ib_prctl_set(task, ctrl); |
2130 | case PR_SPEC_L1D_FLUSH: |
2131 | return l1d_flush_prctl_set(task, ctrl); |
2132 | default: |
2133 | return -ENODEV; |
2134 | } |
2135 | } |
2136 | |
2137 | #ifdef CONFIG_SECCOMP |
2138 | void arch_seccomp_spec_mitigate(struct task_struct *task) |
2139 | { |
2140 | if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) |
2141 | ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); |
2142 | if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || |
2143 | spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) |
2144 | ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); |
2145 | } |
2146 | #endif |
2147 | |
2148 | static int l1d_flush_prctl_get(struct task_struct *task) |
2149 | { |
2150 | if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) |
2151 | return PR_SPEC_FORCE_DISABLE; |
2152 | |
2153 | if (test_ti_thread_flag(ti: &task->thread_info, TIF_SPEC_L1D_FLUSH)) |
2154 | return PR_SPEC_PRCTL | PR_SPEC_ENABLE; |
2155 | else |
2156 | return PR_SPEC_PRCTL | PR_SPEC_DISABLE; |
2157 | } |
2158 | |
2159 | static int ssb_prctl_get(struct task_struct *task) |
2160 | { |
2161 | switch (ssb_mode) { |
2162 | case SPEC_STORE_BYPASS_NONE: |
2163 | if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) |
2164 | return PR_SPEC_ENABLE; |
2165 | return PR_SPEC_NOT_AFFECTED; |
2166 | case SPEC_STORE_BYPASS_DISABLE: |
2167 | return PR_SPEC_DISABLE; |
2168 | case SPEC_STORE_BYPASS_SECCOMP: |
2169 | case SPEC_STORE_BYPASS_PRCTL: |
2170 | if (task_spec_ssb_force_disable(p: task)) |
2171 | return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; |
2172 | if (task_spec_ssb_noexec(p: task)) |
2173 | return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; |
2174 | if (task_spec_ssb_disable(p: task)) |
2175 | return PR_SPEC_PRCTL | PR_SPEC_DISABLE; |
2176 | return PR_SPEC_PRCTL | PR_SPEC_ENABLE; |
2177 | } |
2178 | BUG(); |
2179 | } |
2180 | |
2181 | static int ib_prctl_get(struct task_struct *task) |
2182 | { |
2183 | if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) |
2184 | return PR_SPEC_NOT_AFFECTED; |
2185 | |
2186 | if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && |
2187 | spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) |
2188 | return PR_SPEC_ENABLE; |
2189 | else if (is_spec_ib_user_controlled()) { |
2190 | if (task_spec_ib_force_disable(p: task)) |
2191 | return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; |
2192 | if (task_spec_ib_disable(p: task)) |
2193 | return PR_SPEC_PRCTL | PR_SPEC_DISABLE; |
2194 | return PR_SPEC_PRCTL | PR_SPEC_ENABLE; |
2195 | } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || |
2196 | spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || |
2197 | spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) |
2198 | return PR_SPEC_DISABLE; |
2199 | else |
2200 | return PR_SPEC_NOT_AFFECTED; |
2201 | } |
2202 | |
2203 | int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) |
2204 | { |
2205 | switch (which) { |
2206 | case PR_SPEC_STORE_BYPASS: |
2207 | return ssb_prctl_get(task); |
2208 | case PR_SPEC_INDIRECT_BRANCH: |
2209 | return ib_prctl_get(task); |
2210 | case PR_SPEC_L1D_FLUSH: |
2211 | return l1d_flush_prctl_get(task); |
2212 | default: |
2213 | return -ENODEV; |
2214 | } |
2215 | } |
2216 | |
2217 | void x86_spec_ctrl_setup_ap(void) |
2218 | { |
2219 | if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) |
2220 | update_spec_ctrl(val: x86_spec_ctrl_base); |
2221 | |
2222 | if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) |
2223 | x86_amd_ssb_disable(); |
2224 | } |
2225 | |
2226 | bool itlb_multihit_kvm_mitigation; |
2227 | EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); |
2228 | |
2229 | #undef pr_fmt |
2230 | #define pr_fmt(fmt) "L1TF: " fmt |
2231 | |
2232 | /* Default mitigation for L1TF-affected CPUs */ |
2233 | enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; |
2234 | #if IS_ENABLED(CONFIG_KVM_INTEL) |
2235 | EXPORT_SYMBOL_GPL(l1tf_mitigation); |
2236 | #endif |
2237 | enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; |
2238 | EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); |
2239 | |
2240 | /* |
2241 | * These CPUs all support 44bits physical address space internally in the |
2242 | * cache but CPUID can report a smaller number of physical address bits. |
2243 | * |
2244 | * The L1TF mitigation uses the top most address bit for the inversion of |
2245 | * non present PTEs. When the installed memory reaches into the top most |
2246 | * address bit due to memory holes, which has been observed on machines |
2247 | * which report 36bits physical address bits and have 32G RAM installed, |
2248 | * then the mitigation range check in l1tf_select_mitigation() triggers. |
2249 | * This is a false positive because the mitigation is still possible due to |
2250 | * the fact that the cache uses 44bit internally. Use the cache bits |
2251 | * instead of the reported physical bits and adjust them on the affected |
2252 | * machines to 44bit if the reported bits are less than 44. |
2253 | */ |
2254 | static void override_cache_bits(struct cpuinfo_x86 *c) |
2255 | { |
2256 | if (c->x86 != 6) |
2257 | return; |
2258 | |
2259 | switch (c->x86_model) { |
2260 | case INTEL_FAM6_NEHALEM: |
2261 | case INTEL_FAM6_WESTMERE: |
2262 | case INTEL_FAM6_SANDYBRIDGE: |
2263 | case INTEL_FAM6_IVYBRIDGE: |
2264 | case INTEL_FAM6_HASWELL: |
2265 | case INTEL_FAM6_HASWELL_L: |
2266 | case INTEL_FAM6_HASWELL_G: |
2267 | case INTEL_FAM6_BROADWELL: |
2268 | case INTEL_FAM6_BROADWELL_G: |
2269 | case INTEL_FAM6_SKYLAKE_L: |
2270 | case INTEL_FAM6_SKYLAKE: |
2271 | case INTEL_FAM6_KABYLAKE_L: |
2272 | case INTEL_FAM6_KABYLAKE: |
2273 | if (c->x86_cache_bits < 44) |
2274 | c->x86_cache_bits = 44; |
2275 | break; |
2276 | } |
2277 | } |
2278 | |
2279 | static void __init l1tf_select_mitigation(void) |
2280 | { |
2281 | u64 half_pa; |
2282 | |
2283 | if (!boot_cpu_has_bug(X86_BUG_L1TF)) |
2284 | return; |
2285 | |
2286 | if (cpu_mitigations_off()) |
2287 | l1tf_mitigation = L1TF_MITIGATION_OFF; |
2288 | else if (cpu_mitigations_auto_nosmt()) |
2289 | l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; |
2290 | |
2291 | override_cache_bits(c: &boot_cpu_data); |
2292 | |
2293 | switch (l1tf_mitigation) { |
2294 | case L1TF_MITIGATION_OFF: |
2295 | case L1TF_MITIGATION_FLUSH_NOWARN: |
2296 | case L1TF_MITIGATION_FLUSH: |
2297 | break; |
2298 | case L1TF_MITIGATION_FLUSH_NOSMT: |
2299 | case L1TF_MITIGATION_FULL: |
2300 | cpu_smt_disable(force: false); |
2301 | break; |
2302 | case L1TF_MITIGATION_FULL_FORCE: |
2303 | cpu_smt_disable(force: true); |
2304 | break; |
2305 | } |
2306 | |
2307 | #if CONFIG_PGTABLE_LEVELS == 2 |
2308 | pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n" ); |
2309 | return; |
2310 | #endif |
2311 | |
2312 | half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; |
2313 | if (l1tf_mitigation != L1TF_MITIGATION_OFF && |
2314 | e820__mapped_any(start: half_pa, ULLONG_MAX - half_pa, type: E820_TYPE_RAM)) { |
2315 | pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n" ); |
2316 | pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n" , |
2317 | half_pa); |
2318 | pr_info("However, doing so will make a part of your RAM unusable.\n" ); |
2319 | pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n" ); |
2320 | return; |
2321 | } |
2322 | |
2323 | setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); |
2324 | } |
2325 | |
2326 | static int __init l1tf_cmdline(char *str) |
2327 | { |
2328 | if (!boot_cpu_has_bug(X86_BUG_L1TF)) |
2329 | return 0; |
2330 | |
2331 | if (!str) |
2332 | return -EINVAL; |
2333 | |
2334 | if (!strcmp(str, "off" )) |
2335 | l1tf_mitigation = L1TF_MITIGATION_OFF; |
2336 | else if (!strcmp(str, "flush,nowarn" )) |
2337 | l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; |
2338 | else if (!strcmp(str, "flush" )) |
2339 | l1tf_mitigation = L1TF_MITIGATION_FLUSH; |
2340 | else if (!strcmp(str, "flush,nosmt" )) |
2341 | l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; |
2342 | else if (!strcmp(str, "full" )) |
2343 | l1tf_mitigation = L1TF_MITIGATION_FULL; |
2344 | else if (!strcmp(str, "full,force" )) |
2345 | l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; |
2346 | |
2347 | return 0; |
2348 | } |
2349 | early_param("l1tf" , l1tf_cmdline); |
2350 | |
2351 | #undef pr_fmt |
2352 | #define pr_fmt(fmt) "Speculative Return Stack Overflow: " fmt |
2353 | |
2354 | enum srso_mitigation { |
2355 | SRSO_MITIGATION_NONE, |
2356 | SRSO_MITIGATION_UCODE_NEEDED, |
2357 | SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED, |
2358 | SRSO_MITIGATION_MICROCODE, |
2359 | SRSO_MITIGATION_SAFE_RET, |
2360 | SRSO_MITIGATION_IBPB, |
2361 | SRSO_MITIGATION_IBPB_ON_VMEXIT, |
2362 | }; |
2363 | |
2364 | enum srso_mitigation_cmd { |
2365 | SRSO_CMD_OFF, |
2366 | SRSO_CMD_MICROCODE, |
2367 | SRSO_CMD_SAFE_RET, |
2368 | SRSO_CMD_IBPB, |
2369 | SRSO_CMD_IBPB_ON_VMEXIT, |
2370 | }; |
2371 | |
2372 | static const char * const srso_strings[] = { |
2373 | [SRSO_MITIGATION_NONE] = "Vulnerable" , |
2374 | [SRSO_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode" , |
2375 | [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode" , |
2376 | [SRSO_MITIGATION_MICROCODE] = "Vulnerable: Microcode, no safe RET" , |
2377 | [SRSO_MITIGATION_SAFE_RET] = "Mitigation: Safe RET" , |
2378 | [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB" , |
2379 | [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only" |
2380 | }; |
2381 | |
2382 | static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE; |
2383 | static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET; |
2384 | |
2385 | static int __init srso_parse_cmdline(char *str) |
2386 | { |
2387 | if (!str) |
2388 | return -EINVAL; |
2389 | |
2390 | if (!strcmp(str, "off" )) |
2391 | srso_cmd = SRSO_CMD_OFF; |
2392 | else if (!strcmp(str, "microcode" )) |
2393 | srso_cmd = SRSO_CMD_MICROCODE; |
2394 | else if (!strcmp(str, "safe-ret" )) |
2395 | srso_cmd = SRSO_CMD_SAFE_RET; |
2396 | else if (!strcmp(str, "ibpb" )) |
2397 | srso_cmd = SRSO_CMD_IBPB; |
2398 | else if (!strcmp(str, "ibpb-vmexit" )) |
2399 | srso_cmd = SRSO_CMD_IBPB_ON_VMEXIT; |
2400 | else |
2401 | pr_err("Ignoring unknown SRSO option (%s)." , str); |
2402 | |
2403 | return 0; |
2404 | } |
2405 | early_param("spec_rstack_overflow" , srso_parse_cmdline); |
2406 | |
2407 | #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options." |
2408 | |
2409 | static void __init srso_select_mitigation(void) |
2410 | { |
2411 | bool has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE); |
2412 | |
2413 | if (cpu_mitigations_off()) |
2414 | return; |
2415 | |
2416 | if (!boot_cpu_has_bug(X86_BUG_SRSO)) { |
2417 | if (boot_cpu_has(X86_FEATURE_SBPB)) |
2418 | x86_pred_cmd = PRED_CMD_SBPB; |
2419 | return; |
2420 | } |
2421 | |
2422 | if (has_microcode) { |
2423 | /* |
2424 | * Zen1/2 with SMT off aren't vulnerable after the right |
2425 | * IBPB microcode has been applied. |
2426 | * |
2427 | * Zen1/2 don't have SBPB, no need to try to enable it here. |
2428 | */ |
2429 | if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { |
2430 | setup_force_cpu_cap(X86_FEATURE_SRSO_NO); |
2431 | return; |
2432 | } |
2433 | |
2434 | if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { |
2435 | srso_mitigation = SRSO_MITIGATION_IBPB; |
2436 | goto out; |
2437 | } |
2438 | } else { |
2439 | pr_warn("IBPB-extending microcode not applied!\n" ); |
2440 | pr_warn(SRSO_NOTICE); |
2441 | |
2442 | /* may be overwritten by SRSO_CMD_SAFE_RET below */ |
2443 | srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED; |
2444 | } |
2445 | |
2446 | switch (srso_cmd) { |
2447 | case SRSO_CMD_OFF: |
2448 | if (boot_cpu_has(X86_FEATURE_SBPB)) |
2449 | x86_pred_cmd = PRED_CMD_SBPB; |
2450 | return; |
2451 | |
2452 | case SRSO_CMD_MICROCODE: |
2453 | if (has_microcode) { |
2454 | srso_mitigation = SRSO_MITIGATION_MICROCODE; |
2455 | pr_warn(SRSO_NOTICE); |
2456 | } |
2457 | break; |
2458 | |
2459 | case SRSO_CMD_SAFE_RET: |
2460 | if (IS_ENABLED(CONFIG_CPU_SRSO)) { |
2461 | /* |
2462 | * Enable the return thunk for generated code |
2463 | * like ftrace, static_call, etc. |
2464 | */ |
2465 | setup_force_cpu_cap(X86_FEATURE_RETHUNK); |
2466 | setup_force_cpu_cap(X86_FEATURE_UNRET); |
2467 | |
2468 | if (boot_cpu_data.x86 == 0x19) { |
2469 | setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); |
2470 | x86_return_thunk = srso_alias_return_thunk; |
2471 | } else { |
2472 | setup_force_cpu_cap(X86_FEATURE_SRSO); |
2473 | x86_return_thunk = srso_return_thunk; |
2474 | } |
2475 | if (has_microcode) |
2476 | srso_mitigation = SRSO_MITIGATION_SAFE_RET; |
2477 | else |
2478 | srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED; |
2479 | } else { |
2480 | pr_err("WARNING: kernel not compiled with CPU_SRSO.\n" ); |
2481 | } |
2482 | break; |
2483 | |
2484 | case SRSO_CMD_IBPB: |
2485 | if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) { |
2486 | if (has_microcode) { |
2487 | setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); |
2488 | srso_mitigation = SRSO_MITIGATION_IBPB; |
2489 | } |
2490 | } else { |
2491 | pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n" ); |
2492 | } |
2493 | break; |
2494 | |
2495 | case SRSO_CMD_IBPB_ON_VMEXIT: |
2496 | if (IS_ENABLED(CONFIG_CPU_SRSO)) { |
2497 | if (!boot_cpu_has(X86_FEATURE_ENTRY_IBPB) && has_microcode) { |
2498 | setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); |
2499 | srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT; |
2500 | } |
2501 | } else { |
2502 | pr_err("WARNING: kernel not compiled with CPU_SRSO.\n" ); |
2503 | } |
2504 | break; |
2505 | } |
2506 | |
2507 | out: |
2508 | pr_info("%s\n" , srso_strings[srso_mitigation]); |
2509 | } |
2510 | |
2511 | #undef pr_fmt |
2512 | #define pr_fmt(fmt) fmt |
2513 | |
2514 | #ifdef CONFIG_SYSFS |
2515 | |
2516 | #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" |
2517 | |
2518 | #if IS_ENABLED(CONFIG_KVM_INTEL) |
2519 | static const char * const l1tf_vmx_states[] = { |
2520 | [VMENTER_L1D_FLUSH_AUTO] = "auto" , |
2521 | [VMENTER_L1D_FLUSH_NEVER] = "vulnerable" , |
2522 | [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes" , |
2523 | [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes" , |
2524 | [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled" , |
2525 | [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" |
2526 | }; |
2527 | |
2528 | static ssize_t l1tf_show_state(char *buf) |
2529 | { |
2530 | if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) |
2531 | return sysfs_emit(buf, fmt: "%s\n" , L1TF_DEFAULT_MSG); |
2532 | |
2533 | if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || |
2534 | (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && |
2535 | sched_smt_active())) { |
2536 | return sysfs_emit(buf, fmt: "%s; VMX: %s\n" , L1TF_DEFAULT_MSG, |
2537 | l1tf_vmx_states[l1tf_vmx_mitigation]); |
2538 | } |
2539 | |
2540 | return sysfs_emit(buf, fmt: "%s; VMX: %s, SMT %s\n" , L1TF_DEFAULT_MSG, |
2541 | l1tf_vmx_states[l1tf_vmx_mitigation], |
2542 | sched_smt_active() ? "vulnerable" : "disabled" ); |
2543 | } |
2544 | |
2545 | static ssize_t itlb_multihit_show_state(char *buf) |
2546 | { |
2547 | if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || |
2548 | !boot_cpu_has(X86_FEATURE_VMX)) |
2549 | return sysfs_emit(buf, fmt: "KVM: Mitigation: VMX unsupported\n" ); |
2550 | else if (!(cr4_read_shadow() & X86_CR4_VMXE)) |
2551 | return sysfs_emit(buf, fmt: "KVM: Mitigation: VMX disabled\n" ); |
2552 | else if (itlb_multihit_kvm_mitigation) |
2553 | return sysfs_emit(buf, fmt: "KVM: Mitigation: Split huge pages\n" ); |
2554 | else |
2555 | return sysfs_emit(buf, fmt: "KVM: Vulnerable\n" ); |
2556 | } |
2557 | #else |
2558 | static ssize_t l1tf_show_state(char *buf) |
2559 | { |
2560 | return sysfs_emit(buf, "%s\n" , L1TF_DEFAULT_MSG); |
2561 | } |
2562 | |
2563 | static ssize_t itlb_multihit_show_state(char *buf) |
2564 | { |
2565 | return sysfs_emit(buf, "Processor vulnerable\n" ); |
2566 | } |
2567 | #endif |
2568 | |
2569 | static ssize_t mds_show_state(char *buf) |
2570 | { |
2571 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { |
2572 | return sysfs_emit(buf, fmt: "%s; SMT Host state unknown\n" , |
2573 | mds_strings[mds_mitigation]); |
2574 | } |
2575 | |
2576 | if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { |
2577 | return sysfs_emit(buf, fmt: "%s; SMT %s\n" , mds_strings[mds_mitigation], |
2578 | (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : |
2579 | sched_smt_active() ? "mitigated" : "disabled" )); |
2580 | } |
2581 | |
2582 | return sysfs_emit(buf, fmt: "%s; SMT %s\n" , mds_strings[mds_mitigation], |
2583 | sched_smt_active() ? "vulnerable" : "disabled" ); |
2584 | } |
2585 | |
2586 | static ssize_t tsx_async_abort_show_state(char *buf) |
2587 | { |
2588 | if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || |
2589 | (taa_mitigation == TAA_MITIGATION_OFF)) |
2590 | return sysfs_emit(buf, fmt: "%s\n" , taa_strings[taa_mitigation]); |
2591 | |
2592 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { |
2593 | return sysfs_emit(buf, fmt: "%s; SMT Host state unknown\n" , |
2594 | taa_strings[taa_mitigation]); |
2595 | } |
2596 | |
2597 | return sysfs_emit(buf, fmt: "%s; SMT %s\n" , taa_strings[taa_mitigation], |
2598 | sched_smt_active() ? "vulnerable" : "disabled" ); |
2599 | } |
2600 | |
2601 | static ssize_t mmio_stale_data_show_state(char *buf) |
2602 | { |
2603 | if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) |
2604 | return sysfs_emit(buf, fmt: "Unknown: No mitigations\n" ); |
2605 | |
2606 | if (mmio_mitigation == MMIO_MITIGATION_OFF) |
2607 | return sysfs_emit(buf, fmt: "%s\n" , mmio_strings[mmio_mitigation]); |
2608 | |
2609 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { |
2610 | return sysfs_emit(buf, fmt: "%s; SMT Host state unknown\n" , |
2611 | mmio_strings[mmio_mitigation]); |
2612 | } |
2613 | |
2614 | return sysfs_emit(buf, fmt: "%s; SMT %s\n" , mmio_strings[mmio_mitigation], |
2615 | sched_smt_active() ? "vulnerable" : "disabled" ); |
2616 | } |
2617 | |
2618 | static char *stibp_state(void) |
2619 | { |
2620 | if (spectre_v2_in_eibrs_mode(mode: spectre_v2_enabled) && |
2621 | !boot_cpu_has(X86_FEATURE_AUTOIBRS)) |
2622 | return "" ; |
2623 | |
2624 | switch (spectre_v2_user_stibp) { |
2625 | case SPECTRE_V2_USER_NONE: |
2626 | return ", STIBP: disabled" ; |
2627 | case SPECTRE_V2_USER_STRICT: |
2628 | return ", STIBP: forced" ; |
2629 | case SPECTRE_V2_USER_STRICT_PREFERRED: |
2630 | return ", STIBP: always-on" ; |
2631 | case SPECTRE_V2_USER_PRCTL: |
2632 | case SPECTRE_V2_USER_SECCOMP: |
2633 | if (static_key_enabled(&switch_to_cond_stibp)) |
2634 | return ", STIBP: conditional" ; |
2635 | } |
2636 | return "" ; |
2637 | } |
2638 | |
2639 | static char *ibpb_state(void) |
2640 | { |
2641 | if (boot_cpu_has(X86_FEATURE_IBPB)) { |
2642 | if (static_key_enabled(&switch_mm_always_ibpb)) |
2643 | return ", IBPB: always-on" ; |
2644 | if (static_key_enabled(&switch_mm_cond_ibpb)) |
2645 | return ", IBPB: conditional" ; |
2646 | return ", IBPB: disabled" ; |
2647 | } |
2648 | return "" ; |
2649 | } |
2650 | |
2651 | static char *pbrsb_eibrs_state(void) |
2652 | { |
2653 | if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { |
2654 | if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || |
2655 | boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) |
2656 | return ", PBRSB-eIBRS: SW sequence" ; |
2657 | else |
2658 | return ", PBRSB-eIBRS: Vulnerable" ; |
2659 | } else { |
2660 | return ", PBRSB-eIBRS: Not affected" ; |
2661 | } |
2662 | } |
2663 | |
2664 | static ssize_t spectre_v2_show_state(char *buf) |
2665 | { |
2666 | if (spectre_v2_enabled == SPECTRE_V2_LFENCE) |
2667 | return sysfs_emit(buf, fmt: "Vulnerable: LFENCE\n" ); |
2668 | |
2669 | if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) |
2670 | return sysfs_emit(buf, fmt: "Vulnerable: eIBRS with unprivileged eBPF\n" ); |
2671 | |
2672 | if (sched_smt_active() && unprivileged_ebpf_enabled() && |
2673 | spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) |
2674 | return sysfs_emit(buf, fmt: "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n" ); |
2675 | |
2676 | return sysfs_emit(buf, fmt: "%s%s%s%s%s%s%s\n" , |
2677 | spectre_v2_strings[spectre_v2_enabled], |
2678 | ibpb_state(), |
2679 | boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "" , |
2680 | stibp_state(), |
2681 | boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "" , |
2682 | pbrsb_eibrs_state(), |
2683 | spectre_v2_module_string()); |
2684 | } |
2685 | |
2686 | static ssize_t srbds_show_state(char *buf) |
2687 | { |
2688 | return sysfs_emit(buf, fmt: "%s\n" , srbds_strings[srbds_mitigation]); |
2689 | } |
2690 | |
2691 | static ssize_t retbleed_show_state(char *buf) |
2692 | { |
2693 | if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || |
2694 | retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { |
2695 | if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && |
2696 | boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) |
2697 | return sysfs_emit(buf, fmt: "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n" ); |
2698 | |
2699 | return sysfs_emit(buf, fmt: "%s; SMT %s\n" , retbleed_strings[retbleed_mitigation], |
2700 | !sched_smt_active() ? "disabled" : |
2701 | spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || |
2702 | spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? |
2703 | "enabled with STIBP protection" : "vulnerable" ); |
2704 | } |
2705 | |
2706 | return sysfs_emit(buf, fmt: "%s\n" , retbleed_strings[retbleed_mitigation]); |
2707 | } |
2708 | |
2709 | static ssize_t srso_show_state(char *buf) |
2710 | { |
2711 | if (boot_cpu_has(X86_FEATURE_SRSO_NO)) |
2712 | return sysfs_emit(buf, fmt: "Mitigation: SMT disabled\n" ); |
2713 | |
2714 | return sysfs_emit(buf, fmt: "%s\n" , srso_strings[srso_mitigation]); |
2715 | } |
2716 | |
2717 | static ssize_t gds_show_state(char *buf) |
2718 | { |
2719 | return sysfs_emit(buf, fmt: "%s\n" , gds_strings[gds_mitigation]); |
2720 | } |
2721 | |
2722 | static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, |
2723 | char *buf, unsigned int bug) |
2724 | { |
2725 | if (!boot_cpu_has_bug(bug)) |
2726 | return sysfs_emit(buf, fmt: "Not affected\n" ); |
2727 | |
2728 | switch (bug) { |
2729 | case X86_BUG_CPU_MELTDOWN: |
2730 | if (boot_cpu_has(X86_FEATURE_PTI)) |
2731 | return sysfs_emit(buf, fmt: "Mitigation: PTI\n" ); |
2732 | |
2733 | if (hypervisor_is_type(type: X86_HYPER_XEN_PV)) |
2734 | return sysfs_emit(buf, fmt: "Unknown (XEN PV detected, hypervisor mitigation required)\n" ); |
2735 | |
2736 | break; |
2737 | |
2738 | case X86_BUG_SPECTRE_V1: |
2739 | return sysfs_emit(buf, fmt: "%s\n" , spectre_v1_strings[spectre_v1_mitigation]); |
2740 | |
2741 | case X86_BUG_SPECTRE_V2: |
2742 | return spectre_v2_show_state(buf); |
2743 | |
2744 | case X86_BUG_SPEC_STORE_BYPASS: |
2745 | return sysfs_emit(buf, fmt: "%s\n" , ssb_strings[ssb_mode]); |
2746 | |
2747 | case X86_BUG_L1TF: |
2748 | if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) |
2749 | return l1tf_show_state(buf); |
2750 | break; |
2751 | |
2752 | case X86_BUG_MDS: |
2753 | return mds_show_state(buf); |
2754 | |
2755 | case X86_BUG_TAA: |
2756 | return tsx_async_abort_show_state(buf); |
2757 | |
2758 | case X86_BUG_ITLB_MULTIHIT: |
2759 | return itlb_multihit_show_state(buf); |
2760 | |
2761 | case X86_BUG_SRBDS: |
2762 | return srbds_show_state(buf); |
2763 | |
2764 | case X86_BUG_MMIO_STALE_DATA: |
2765 | case X86_BUG_MMIO_UNKNOWN: |
2766 | return mmio_stale_data_show_state(buf); |
2767 | |
2768 | case X86_BUG_RETBLEED: |
2769 | return retbleed_show_state(buf); |
2770 | |
2771 | case X86_BUG_SRSO: |
2772 | return srso_show_state(buf); |
2773 | |
2774 | case X86_BUG_GDS: |
2775 | return gds_show_state(buf); |
2776 | |
2777 | default: |
2778 | break; |
2779 | } |
2780 | |
2781 | return sysfs_emit(buf, fmt: "Vulnerable\n" ); |
2782 | } |
2783 | |
2784 | ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) |
2785 | { |
2786 | return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); |
2787 | } |
2788 | |
2789 | ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) |
2790 | { |
2791 | return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); |
2792 | } |
2793 | |
2794 | ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) |
2795 | { |
2796 | return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); |
2797 | } |
2798 | |
2799 | ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) |
2800 | { |
2801 | return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); |
2802 | } |
2803 | |
2804 | ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) |
2805 | { |
2806 | return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); |
2807 | } |
2808 | |
2809 | ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) |
2810 | { |
2811 | return cpu_show_common(dev, attr, buf, X86_BUG_MDS); |
2812 | } |
2813 | |
2814 | ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) |
2815 | { |
2816 | return cpu_show_common(dev, attr, buf, X86_BUG_TAA); |
2817 | } |
2818 | |
2819 | ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) |
2820 | { |
2821 | return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); |
2822 | } |
2823 | |
2824 | ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) |
2825 | { |
2826 | return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); |
2827 | } |
2828 | |
2829 | ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) |
2830 | { |
2831 | if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) |
2832 | return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN); |
2833 | else |
2834 | return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); |
2835 | } |
2836 | |
2837 | ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) |
2838 | { |
2839 | return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); |
2840 | } |
2841 | |
2842 | ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf) |
2843 | { |
2844 | return cpu_show_common(dev, attr, buf, X86_BUG_SRSO); |
2845 | } |
2846 | |
2847 | ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf) |
2848 | { |
2849 | return cpu_show_common(dev, attr, buf, X86_BUG_GDS); |
2850 | } |
2851 | #endif |
2852 | |