1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * x86 FPU boot time init code:
4 */
5#include <asm/fpu/api.h>
6#include <asm/tlbflush.h>
7#include <asm/setup.h>
8
9#include <linux/sched.h>
10#include <linux/sched/task.h>
11#include <linux/init.h>
12
13#include "internal.h"
14#include "legacy.h"
15#include "xstate.h"
16
17/*
18 * Initialize the registers found in all CPUs, CR0 and CR4:
19 */
20static void fpu__init_cpu_generic(void)
21{
22 unsigned long cr0;
23 unsigned long cr4_mask = 0;
24
25 if (boot_cpu_has(X86_FEATURE_FXSR))
26 cr4_mask |= X86_CR4_OSFXSR;
27 if (boot_cpu_has(X86_FEATURE_XMM))
28 cr4_mask |= X86_CR4_OSXMMEXCPT;
29 if (cr4_mask)
30 cr4_set_bits(mask: cr4_mask);
31
32 cr0 = read_cr0();
33 cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
34 if (!boot_cpu_has(X86_FEATURE_FPU))
35 cr0 |= X86_CR0_EM;
36 write_cr0(x: cr0);
37
38 /* Flush out any pending x87 state: */
39#ifdef CONFIG_MATH_EMULATION
40 if (!boot_cpu_has(X86_FEATURE_FPU))
41 ;
42 else
43#endif
44 asm volatile ("fninit");
45}
46
47/*
48 * Enable all supported FPU features. Called when a CPU is brought online:
49 */
50void fpu__init_cpu(void)
51{
52 fpu__init_cpu_generic();
53 fpu__init_cpu_xstate();
54
55 /* Start allowing kernel-mode FPU: */
56 this_cpu_write(kernel_fpu_allowed, true);
57}
58
59static bool __init fpu__probe_without_cpuid(void)
60{
61 unsigned long cr0;
62 u16 fsw, fcw;
63
64 fsw = fcw = 0xffff;
65
66 cr0 = read_cr0();
67 cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
68 write_cr0(x: cr0);
69
70 asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw));
71
72 pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw);
73
74 return fsw == 0 && (fcw & 0x103f) == 0x003f;
75}
76
77static void __init fpu__init_system_early_generic(void)
78{
79 set_thread_flag(TIF_NEED_FPU_LOAD);
80
81 if (!boot_cpu_has(X86_FEATURE_CPUID) &&
82 !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {
83 if (fpu__probe_without_cpuid())
84 setup_force_cpu_cap(X86_FEATURE_FPU);
85 else
86 setup_clear_cpu_cap(X86_FEATURE_FPU);
87 }
88
89#ifndef CONFIG_MATH_EMULATION
90 if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
91 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
92 for (;;)
93 asm volatile("hlt");
94 }
95#endif
96}
97
98/*
99 * Boot time FPU feature detection code:
100 */
101unsigned int mxcsr_feature_mask __ro_after_init = 0xffffffffu;
102
103static void __init fpu__init_system_mxcsr(void)
104{
105 unsigned int mask = 0;
106
107 if (boot_cpu_has(X86_FEATURE_FXSR)) {
108 /* Static because GCC does not get 16-byte stack alignment right: */
109 static struct fxregs_state fxregs __initdata;
110
111 asm volatile("fxsave %0" : "+m" (fxregs));
112
113 mask = fxregs.mxcsr_mask;
114
115 /*
116 * If zero then use the default features mask,
117 * which has all features set, except the
118 * denormals-are-zero feature bit:
119 */
120 if (mask == 0)
121 mask = 0x0000ffbf;
122 }
123 mxcsr_feature_mask &= mask;
124}
125
126/*
127 * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
128 */
129static void __init fpu__init_system_generic(void)
130{
131 /*
132 * Set up the legacy init FPU context. Will be updated when the
133 * CPU supports XSAVE[S].
134 */
135 fpstate_init_user(fpstate: &init_fpstate);
136
137 fpu__init_system_mxcsr();
138}
139
140/*
141 * Enforce that 'MEMBER' is the last field of 'TYPE'.
142 *
143 * Align the computed size with alignment of the TYPE,
144 * because that's how C aligns structs.
145 */
146#define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
147 BUILD_BUG_ON(sizeof(TYPE) != \
148 ALIGN(offsetofend(TYPE, MEMBER), _Alignof(TYPE)))
149
150/*
151 * We append the 'struct fpu' to the task_struct:
152 */
153static void __init fpu__init_task_struct_size(void)
154{
155 int task_size = sizeof(struct task_struct);
156
157 task_size += sizeof(struct fpu);
158
159 /*
160 * Subtract off the static size of the register state.
161 * It potentially has a bunch of padding.
162 */
163 task_size -= sizeof(union fpregs_state);
164
165 /*
166 * Add back the dynamically-calculated register state
167 * size.
168 */
169 task_size += fpu_kernel_cfg.default_size;
170
171 /*
172 * We dynamically size 'struct fpu', so we require that
173 * 'state' be at the end of 'it:
174 */
175 CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate);
176
177 arch_task_struct_size = task_size;
178}
179
180/*
181 * Set up the user and kernel xstate sizes based on the legacy FPU context size.
182 *
183 * We set this up first, and later it will be overwritten by
184 * fpu__init_system_xstate() if the CPU knows about xstates.
185 */
186static void __init fpu__init_system_xstate_size_legacy(void)
187{
188 unsigned int size;
189
190 /*
191 * Note that the size configuration might be overwritten later
192 * during fpu__init_system_xstate().
193 */
194 if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
195 size = sizeof(struct swregs_state);
196 } else if (cpu_feature_enabled(X86_FEATURE_FXSR)) {
197 size = sizeof(struct fxregs_state);
198 fpu_user_cfg.legacy_features = XFEATURE_MASK_FPSSE;
199 } else {
200 size = sizeof(struct fregs_state);
201 fpu_user_cfg.legacy_features = XFEATURE_MASK_FP;
202 }
203
204 fpu_kernel_cfg.max_size = size;
205 fpu_kernel_cfg.default_size = size;
206 fpu_user_cfg.max_size = size;
207 fpu_user_cfg.default_size = size;
208 guest_default_cfg.size = size;
209}
210
211/*
212 * Called on the boot CPU once per system bootup, to set up the initial
213 * FPU state that is later cloned into all processes:
214 */
215void __init fpu__init_system(void)
216{
217 fpu__init_system_early_generic();
218
219 /*
220 * The FPU has to be operational for some of the
221 * later FPU init activities:
222 */
223 fpu__init_cpu();
224
225 fpu__init_system_generic();
226 fpu__init_system_xstate_size_legacy();
227 fpu__init_system_xstate(legacy_size: fpu_kernel_cfg.max_size);
228 fpu__init_task_struct_size();
229}
230

source code of linux/arch/x86/kernel/fpu/init.c