1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Intel Transactional Synchronization Extensions (TSX) control. |
4 | * |
5 | * Copyright (C) 2019-2021 Intel Corporation |
6 | * |
7 | * Author: |
8 | * Pawan Gupta <pawan.kumar.gupta@linux.intel.com> |
9 | */ |
10 | |
11 | #include <linux/cpufeature.h> |
12 | |
13 | #include <asm/cmdline.h> |
14 | #include <asm/cpu.h> |
15 | |
16 | #include "cpu.h" |
17 | |
18 | #undef pr_fmt |
19 | #define pr_fmt(fmt) "tsx: " fmt |
20 | |
21 | enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED; |
22 | |
23 | static void tsx_disable(void) |
24 | { |
25 | u64 tsx; |
26 | |
27 | rdmsrl(MSR_IA32_TSX_CTRL, tsx); |
28 | |
29 | /* Force all transactions to immediately abort */ |
30 | tsx |= TSX_CTRL_RTM_DISABLE; |
31 | |
32 | /* |
33 | * Ensure TSX support is not enumerated in CPUID. |
34 | * This is visible to userspace and will ensure they |
35 | * do not waste resources trying TSX transactions that |
36 | * will always abort. |
37 | */ |
38 | tsx |= TSX_CTRL_CPUID_CLEAR; |
39 | |
40 | wrmsrl(MSR_IA32_TSX_CTRL, val: tsx); |
41 | } |
42 | |
43 | static void tsx_enable(void) |
44 | { |
45 | u64 tsx; |
46 | |
47 | rdmsrl(MSR_IA32_TSX_CTRL, tsx); |
48 | |
49 | /* Enable the RTM feature in the cpu */ |
50 | tsx &= ~TSX_CTRL_RTM_DISABLE; |
51 | |
52 | /* |
53 | * Ensure TSX support is enumerated in CPUID. |
54 | * This is visible to userspace and will ensure they |
55 | * can enumerate and use the TSX feature. |
56 | */ |
57 | tsx &= ~TSX_CTRL_CPUID_CLEAR; |
58 | |
59 | wrmsrl(MSR_IA32_TSX_CTRL, val: tsx); |
60 | } |
61 | |
62 | static enum tsx_ctrl_states x86_get_tsx_auto_mode(void) |
63 | { |
64 | if (boot_cpu_has_bug(X86_BUG_TAA)) |
65 | return TSX_CTRL_DISABLE; |
66 | |
67 | return TSX_CTRL_ENABLE; |
68 | } |
69 | |
70 | /* |
71 | * Disabling TSX is not a trivial business. |
72 | * |
73 | * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT |
74 | * which says that TSX is practically disabled (all transactions are |
75 | * aborted by default). When that bit is set, the kernel unconditionally |
76 | * disables TSX. |
77 | * |
78 | * In order to do that, however, it needs to dance a bit: |
79 | * |
80 | * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and |
81 | * the MSR is present only when *two* CPUID bits are set: |
82 | * |
83 | * - X86_FEATURE_RTM_ALWAYS_ABORT |
84 | * - X86_FEATURE_TSX_FORCE_ABORT |
85 | * |
86 | * 2. The second method is for CPUs which do not have the above-mentioned |
87 | * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX |
88 | * through that one. Those CPUs can also have the initially mentioned |
89 | * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy |
90 | * applies: TSX gets disabled unconditionally. |
91 | * |
92 | * When either of the two methods are present, the kernel disables TSX and |
93 | * clears the respective RTM and HLE feature flags. |
94 | * |
95 | * An additional twist in the whole thing presents late microcode loading |
96 | * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID |
97 | * bit to be set after the update. |
98 | * |
99 | * A subsequent hotplug operation on any logical CPU except the BSP will |
100 | * cause for the supported CPUID feature bits to get re-detected and, if |
101 | * RTM and HLE get cleared all of a sudden, but, userspace did consult |
102 | * them before the update, then funny explosions will happen. Long story |
103 | * short: the kernel doesn't modify CPUID feature bits after booting. |
104 | * |
105 | * That's why, this function's call in init_intel() doesn't clear the |
106 | * feature flags. |
107 | */ |
108 | static void tsx_clear_cpuid(void) |
109 | { |
110 | u64 msr; |
111 | |
112 | /* |
113 | * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID |
114 | * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present. |
115 | */ |
116 | if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) && |
117 | boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) { |
118 | rdmsrl(MSR_TSX_FORCE_ABORT, msr); |
119 | msr |= MSR_TFA_TSX_CPUID_CLEAR; |
120 | wrmsrl(MSR_TSX_FORCE_ABORT, val: msr); |
121 | } else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) { |
122 | rdmsrl(MSR_IA32_TSX_CTRL, msr); |
123 | msr |= TSX_CTRL_CPUID_CLEAR; |
124 | wrmsrl(MSR_IA32_TSX_CTRL, val: msr); |
125 | } |
126 | } |
127 | |
128 | /* |
129 | * Disable TSX development mode |
130 | * |
131 | * When the microcode released in Feb 2022 is applied, TSX will be disabled by |
132 | * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123 |
133 | * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is |
134 | * not recommended for production deployments. In particular, applying MD_CLEAR |
135 | * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient |
136 | * execution attack may not be effective on these processors when Intel TSX is |
137 | * enabled with updated microcode. |
138 | */ |
139 | static void tsx_dev_mode_disable(void) |
140 | { |
141 | u64 mcu_opt_ctrl; |
142 | |
143 | /* Check if RTM_ALLOW exists */ |
144 | if (!boot_cpu_has_bug(X86_BUG_TAA) || |
145 | !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) || |
146 | !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL)) |
147 | return; |
148 | |
149 | rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl); |
150 | |
151 | if (mcu_opt_ctrl & RTM_ALLOW) { |
152 | mcu_opt_ctrl &= ~RTM_ALLOW; |
153 | wrmsrl(MSR_IA32_MCU_OPT_CTRL, val: mcu_opt_ctrl); |
154 | setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT); |
155 | } |
156 | } |
157 | |
158 | void __init tsx_init(void) |
159 | { |
160 | char arg[5] = {}; |
161 | int ret; |
162 | |
163 | tsx_dev_mode_disable(); |
164 | |
165 | /* |
166 | * Hardware will always abort a TSX transaction when the CPUID bit |
167 | * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate |
168 | * CPUID.RTM and CPUID.HLE bits. Clear them here. |
169 | */ |
170 | if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) { |
171 | tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT; |
172 | tsx_clear_cpuid(); |
173 | setup_clear_cpu_cap(X86_FEATURE_RTM); |
174 | setup_clear_cpu_cap(X86_FEATURE_HLE); |
175 | return; |
176 | } |
177 | |
178 | /* |
179 | * TSX is controlled via MSR_IA32_TSX_CTRL. However, support for this |
180 | * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES. |
181 | * |
182 | * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a |
183 | * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES |
184 | * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get |
185 | * MSR_IA32_TSX_CTRL support even after a microcode update. Thus, |
186 | * tsx= cmdline requests will do nothing on CPUs without |
187 | * MSR_IA32_TSX_CTRL support. |
188 | */ |
189 | if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) { |
190 | setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL); |
191 | } else { |
192 | tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED; |
193 | return; |
194 | } |
195 | |
196 | ret = cmdline_find_option(cmdline_ptr: boot_command_line, option: "tsx" , buffer: arg, bufsize: sizeof(arg)); |
197 | if (ret >= 0) { |
198 | if (!strcmp(arg, "on" )) { |
199 | tsx_ctrl_state = TSX_CTRL_ENABLE; |
200 | } else if (!strcmp(arg, "off" )) { |
201 | tsx_ctrl_state = TSX_CTRL_DISABLE; |
202 | } else if (!strcmp(arg, "auto" )) { |
203 | tsx_ctrl_state = x86_get_tsx_auto_mode(); |
204 | } else { |
205 | tsx_ctrl_state = TSX_CTRL_DISABLE; |
206 | pr_err("invalid option, defaulting to off\n" ); |
207 | } |
208 | } else { |
209 | /* tsx= not provided */ |
210 | if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO)) |
211 | tsx_ctrl_state = x86_get_tsx_auto_mode(); |
212 | else if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF)) |
213 | tsx_ctrl_state = TSX_CTRL_DISABLE; |
214 | else |
215 | tsx_ctrl_state = TSX_CTRL_ENABLE; |
216 | } |
217 | |
218 | if (tsx_ctrl_state == TSX_CTRL_DISABLE) { |
219 | tsx_disable(); |
220 | |
221 | /* |
222 | * tsx_disable() will change the state of the RTM and HLE CPUID |
223 | * bits. Clear them here since they are now expected to be not |
224 | * set. |
225 | */ |
226 | setup_clear_cpu_cap(X86_FEATURE_RTM); |
227 | setup_clear_cpu_cap(X86_FEATURE_HLE); |
228 | } else if (tsx_ctrl_state == TSX_CTRL_ENABLE) { |
229 | |
230 | /* |
231 | * HW defaults TSX to be enabled at bootup. |
232 | * We may still need the TSX enable support |
233 | * during init for special cases like |
234 | * kexec after TSX is disabled. |
235 | */ |
236 | tsx_enable(); |
237 | |
238 | /* |
239 | * tsx_enable() will change the state of the RTM and HLE CPUID |
240 | * bits. Force them here since they are now expected to be set. |
241 | */ |
242 | setup_force_cpu_cap(X86_FEATURE_RTM); |
243 | setup_force_cpu_cap(X86_FEATURE_HLE); |
244 | } |
245 | } |
246 | |
247 | void tsx_ap_init(void) |
248 | { |
249 | tsx_dev_mode_disable(); |
250 | |
251 | if (tsx_ctrl_state == TSX_CTRL_ENABLE) |
252 | tsx_enable(); |
253 | else if (tsx_ctrl_state == TSX_CTRL_DISABLE) |
254 | tsx_disable(); |
255 | else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT) |
256 | /* See comment over that function for more details. */ |
257 | tsx_clear_cpuid(); |
258 | } |
259 | |