1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * FPU data structures: |
4 | */ |
5 | #ifndef _ASM_X86_FPU_H |
6 | #define _ASM_X86_FPU_H |
7 | |
8 | /* |
9 | * The legacy x87 FPU state format, as saved by FSAVE and |
10 | * restored by the FRSTOR instructions: |
11 | */ |
12 | struct fregs_state { |
13 | u32 cwd; /* FPU Control Word */ |
14 | u32 swd; /* FPU Status Word */ |
15 | u32 twd; /* FPU Tag Word */ |
16 | u32 fip; /* FPU IP Offset */ |
17 | u32 fcs; /* FPU IP Selector */ |
18 | u32 foo; /* FPU Operand Pointer Offset */ |
19 | u32 fos; /* FPU Operand Pointer Selector */ |
20 | |
21 | /* 8*10 bytes for each FP-reg = 80 bytes: */ |
22 | u32 st_space[20]; |
23 | |
24 | /* Software status information [not touched by FSAVE]: */ |
25 | u32 status; |
26 | }; |
27 | |
28 | /* |
29 | * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and |
30 | * restored by the FXRSTOR instructions. It's similar to the FSAVE |
31 | * format, but differs in some areas, plus has extensions at |
32 | * the end for the XMM registers. |
33 | */ |
34 | struct fxregs_state { |
35 | u16 cwd; /* Control Word */ |
36 | u16 swd; /* Status Word */ |
37 | u16 twd; /* Tag Word */ |
38 | u16 fop; /* Last Instruction Opcode */ |
39 | union { |
40 | struct { |
41 | u64 rip; /* Instruction Pointer */ |
42 | u64 rdp; /* Data Pointer */ |
43 | }; |
44 | struct { |
45 | u32 fip; /* FPU IP Offset */ |
46 | u32 fcs; /* FPU IP Selector */ |
47 | u32 foo; /* FPU Operand Offset */ |
48 | u32 fos; /* FPU Operand Selector */ |
49 | }; |
50 | }; |
51 | u32 mxcsr; /* MXCSR Register State */ |
52 | u32 mxcsr_mask; /* MXCSR Mask */ |
53 | |
54 | /* 8*16 bytes for each FP-reg = 128 bytes: */ |
55 | u32 st_space[32]; |
56 | |
57 | /* 16*16 bytes for each XMM-reg = 256 bytes: */ |
58 | u32 xmm_space[64]; |
59 | |
60 | u32 padding[12]; |
61 | |
62 | union { |
63 | u32 padding1[12]; |
64 | u32 sw_reserved[12]; |
65 | }; |
66 | |
67 | } __attribute__((aligned(16))); |
68 | |
69 | /* Default value for fxregs_state.mxcsr: */ |
70 | #define MXCSR_DEFAULT 0x1f80 |
71 | |
72 | /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */ |
73 | #define MXCSR_AND_FLAGS_SIZE sizeof(u64) |
74 | |
75 | /* |
76 | * Software based FPU emulation state. This is arbitrary really, |
77 | * it matches the x87 format to make it easier to understand: |
78 | */ |
79 | struct swregs_state { |
80 | u32 cwd; |
81 | u32 swd; |
82 | u32 twd; |
83 | u32 fip; |
84 | u32 fcs; |
85 | u32 foo; |
86 | u32 fos; |
87 | /* 8*10 bytes for each FP-reg = 80 bytes: */ |
88 | u32 st_space[20]; |
89 | u8 ftop; |
90 | u8 changed; |
91 | u8 lookahead; |
92 | u8 no_update; |
93 | u8 rm; |
94 | u8 alimit; |
95 | struct math_emu_info *info; |
96 | u32 entry_eip; |
97 | }; |
98 | |
99 | /* |
100 | * List of XSAVE features Linux knows about: |
101 | */ |
102 | enum xfeature { |
103 | XFEATURE_FP, |
104 | XFEATURE_SSE, |
105 | /* |
106 | * Values above here are "legacy states". |
107 | * Those below are "extended states". |
108 | */ |
109 | XFEATURE_YMM, |
110 | XFEATURE_BNDREGS, |
111 | XFEATURE_BNDCSR, |
112 | XFEATURE_OPMASK, |
113 | XFEATURE_ZMM_Hi256, |
114 | XFEATURE_Hi16_ZMM, |
115 | XFEATURE_PT_UNIMPLEMENTED_SO_FAR, |
116 | XFEATURE_PKRU, |
117 | XFEATURE_PASID, |
118 | XFEATURE_CET_USER, |
119 | XFEATURE_CET_KERNEL_UNUSED, |
120 | XFEATURE_RSRVD_COMP_13, |
121 | XFEATURE_RSRVD_COMP_14, |
122 | XFEATURE_LBR, |
123 | XFEATURE_RSRVD_COMP_16, |
124 | XFEATURE_XTILE_CFG, |
125 | XFEATURE_XTILE_DATA, |
126 | |
127 | XFEATURE_MAX, |
128 | }; |
129 | |
130 | #define XFEATURE_MASK_FP (1 << XFEATURE_FP) |
131 | #define XFEATURE_MASK_SSE (1 << XFEATURE_SSE) |
132 | #define XFEATURE_MASK_YMM (1 << XFEATURE_YMM) |
133 | #define XFEATURE_MASK_BNDREGS (1 << XFEATURE_BNDREGS) |
134 | #define XFEATURE_MASK_BNDCSR (1 << XFEATURE_BNDCSR) |
135 | #define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) |
136 | #define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) |
137 | #define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) |
138 | #define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR) |
139 | #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) |
140 | #define XFEATURE_MASK_PASID (1 << XFEATURE_PASID) |
141 | #define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER) |
142 | #define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL_UNUSED) |
143 | #define XFEATURE_MASK_LBR (1 << XFEATURE_LBR) |
144 | #define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG) |
145 | #define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA) |
146 | |
147 | #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) |
148 | #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ |
149 | | XFEATURE_MASK_ZMM_Hi256 \ |
150 | | XFEATURE_MASK_Hi16_ZMM) |
151 | |
152 | #ifdef CONFIG_X86_64 |
153 | # define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA \ |
154 | | XFEATURE_MASK_XTILE_CFG) |
155 | #else |
156 | # define XFEATURE_MASK_XTILE (0) |
157 | #endif |
158 | |
159 | #define FIRST_EXTENDED_XFEATURE XFEATURE_YMM |
160 | |
161 | struct reg_128_bit { |
162 | u8 regbytes[128/8]; |
163 | }; |
164 | struct reg_256_bit { |
165 | u8 regbytes[256/8]; |
166 | }; |
167 | struct reg_512_bit { |
168 | u8 regbytes[512/8]; |
169 | }; |
170 | struct reg_1024_byte { |
171 | u8 regbytes[1024]; |
172 | }; |
173 | |
174 | /* |
175 | * State component 2: |
176 | * |
177 | * There are 16x 256-bit AVX registers named YMM0-YMM15. |
178 | * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15) |
179 | * and are stored in 'struct fxregs_state::xmm_space[]' in the |
180 | * "legacy" area. |
181 | * |
182 | * The high 128 bits are stored here. |
183 | */ |
184 | struct ymmh_struct { |
185 | struct reg_128_bit hi_ymm[16]; |
186 | } __packed; |
187 | |
188 | /* Intel MPX support: */ |
189 | |
190 | struct mpx_bndreg { |
191 | u64 lower_bound; |
192 | u64 upper_bound; |
193 | } __packed; |
194 | /* |
195 | * State component 3 is used for the 4 128-bit bounds registers |
196 | */ |
197 | struct mpx_bndreg_state { |
198 | struct mpx_bndreg bndreg[4]; |
199 | } __packed; |
200 | |
201 | /* |
202 | * State component 4 is used for the 64-bit user-mode MPX |
203 | * configuration register BNDCFGU and the 64-bit MPX status |
204 | * register BNDSTATUS. We call the pair "BNDCSR". |
205 | */ |
206 | struct mpx_bndcsr { |
207 | u64 bndcfgu; |
208 | u64 bndstatus; |
209 | } __packed; |
210 | |
211 | /* |
212 | * The BNDCSR state is padded out to be 64-bytes in size. |
213 | */ |
214 | struct mpx_bndcsr_state { |
215 | union { |
216 | struct mpx_bndcsr bndcsr; |
217 | u8 pad_to_64_bytes[64]; |
218 | }; |
219 | } __packed; |
220 | |
221 | /* AVX-512 Components: */ |
222 | |
223 | /* |
224 | * State component 5 is used for the 8 64-bit opmask registers |
225 | * k0-k7 (opmask state). |
226 | */ |
227 | struct avx_512_opmask_state { |
228 | u64 opmask_reg[8]; |
229 | } __packed; |
230 | |
231 | /* |
232 | * State component 6 is used for the upper 256 bits of the |
233 | * registers ZMM0-ZMM15. These 16 256-bit values are denoted |
234 | * ZMM0_H-ZMM15_H (ZMM_Hi256 state). |
235 | */ |
236 | struct avx_512_zmm_uppers_state { |
237 | struct reg_256_bit zmm_upper[16]; |
238 | } __packed; |
239 | |
240 | /* |
241 | * State component 7 is used for the 16 512-bit registers |
242 | * ZMM16-ZMM31 (Hi16_ZMM state). |
243 | */ |
244 | struct avx_512_hi16_state { |
245 | struct reg_512_bit hi16_zmm[16]; |
246 | } __packed; |
247 | |
248 | /* |
249 | * State component 9: 32-bit PKRU register. The state is |
250 | * 8 bytes long but only 4 bytes is used currently. |
251 | */ |
252 | struct pkru_state { |
253 | u32 pkru; |
254 | u32 pad; |
255 | } __packed; |
256 | |
257 | /* |
258 | * State component 11 is Control-flow Enforcement user states |
259 | */ |
260 | struct cet_user_state { |
261 | /* user control-flow settings */ |
262 | u64 user_cet; |
263 | /* user shadow stack pointer */ |
264 | u64 user_ssp; |
265 | }; |
266 | |
267 | /* |
268 | * State component 15: Architectural LBR configuration state. |
269 | * The size of Arch LBR state depends on the number of LBRs (lbr_depth). |
270 | */ |
271 | |
272 | struct lbr_entry { |
273 | u64 from; |
274 | u64 to; |
275 | u64 info; |
276 | }; |
277 | |
278 | struct arch_lbr_state { |
279 | u64 lbr_ctl; |
280 | u64 lbr_depth; |
281 | u64 ler_from; |
282 | u64 ler_to; |
283 | u64 ler_info; |
284 | struct lbr_entry entries[]; |
285 | }; |
286 | |
287 | /* |
288 | * State component 17: 64-byte tile configuration register. |
289 | */ |
290 | struct xtile_cfg { |
291 | u64 tcfg[8]; |
292 | } __packed; |
293 | |
294 | /* |
295 | * State component 18: 1KB tile data register. |
296 | * Each register represents 16 64-byte rows of the matrix |
297 | * data. But the number of registers depends on the actual |
298 | * implementation. |
299 | */ |
300 | struct xtile_data { |
301 | struct reg_1024_byte tmm; |
302 | } __packed; |
303 | |
304 | /* |
305 | * State component 10 is supervisor state used for context-switching the |
306 | * PASID state. |
307 | */ |
308 | struct ia32_pasid_state { |
309 | u64 pasid; |
310 | } __packed; |
311 | |
312 | struct { |
313 | u64 ; |
314 | u64 ; |
315 | u64 [6]; |
316 | } __attribute__((packed)); |
317 | |
318 | /* |
319 | * xstate_header.xcomp_bv[63] indicates that the extended_state_area |
320 | * is in compacted format. |
321 | */ |
322 | #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63) |
323 | |
324 | /* |
325 | * This is our most modern FPU state format, as saved by the XSAVE |
326 | * and restored by the XRSTOR instructions. |
327 | * |
328 | * It consists of a legacy fxregs portion, an xstate header and |
329 | * subsequent areas as defined by the xstate header. Not all CPUs |
330 | * support all the extensions, so the size of the extended area |
331 | * can vary quite a bit between CPUs. |
332 | */ |
333 | struct xregs_state { |
334 | struct fxregs_state i387; |
335 | struct xstate_header ; |
336 | u8 extended_state_area[]; |
337 | } __attribute__ ((packed, aligned (64))); |
338 | |
339 | /* |
340 | * This is a union of all the possible FPU state formats |
341 | * put together, so that we can pick the right one runtime. |
342 | * |
343 | * The size of the structure is determined by the largest |
344 | * member - which is the xsave area. The padding is there |
345 | * to ensure that statically-allocated task_structs (just |
346 | * the init_task today) have enough space. |
347 | */ |
348 | union fpregs_state { |
349 | struct fregs_state fsave; |
350 | struct fxregs_state fxsave; |
351 | struct swregs_state soft; |
352 | struct xregs_state xsave; |
353 | u8 __padding[PAGE_SIZE]; |
354 | }; |
355 | |
356 | struct fpstate { |
357 | /* @kernel_size: The size of the kernel register image */ |
358 | unsigned int size; |
359 | |
360 | /* @user_size: The size in non-compacted UABI format */ |
361 | unsigned int user_size; |
362 | |
363 | /* @xfeatures: xfeatures for which the storage is sized */ |
364 | u64 xfeatures; |
365 | |
366 | /* @user_xfeatures: xfeatures valid in UABI buffers */ |
367 | u64 user_xfeatures; |
368 | |
369 | /* @xfd: xfeatures disabled to trap userspace use. */ |
370 | u64 xfd; |
371 | |
372 | /* @is_valloc: Indicator for dynamically allocated state */ |
373 | unsigned int is_valloc : 1; |
374 | |
375 | /* @is_guest: Indicator for guest state (KVM) */ |
376 | unsigned int is_guest : 1; |
377 | |
378 | /* |
379 | * @is_confidential: Indicator for KVM confidential mode. |
380 | * The FPU registers are restored by the |
381 | * vmentry firmware from encrypted guest |
382 | * memory. On vmexit the FPU registers are |
383 | * saved by firmware to encrypted guest memory |
384 | * and the registers are scrubbed before |
385 | * returning to the host. So there is no |
386 | * content which is worth saving and restoring. |
387 | * The fpstate has to be there so that |
388 | * preemption and softirq FPU usage works |
389 | * without special casing. |
390 | */ |
391 | unsigned int is_confidential : 1; |
392 | |
393 | /* @in_use: State is in use */ |
394 | unsigned int in_use : 1; |
395 | |
396 | /* @regs: The register state union for all supported formats */ |
397 | union fpregs_state regs; |
398 | |
399 | /* @regs is dynamically sized! Don't add anything after @regs! */ |
400 | } __aligned(64); |
401 | |
402 | #define FPU_GUEST_PERM_LOCKED BIT_ULL(63) |
403 | |
404 | struct fpu_state_perm { |
405 | /* |
406 | * @__state_perm: |
407 | * |
408 | * This bitmap indicates the permission for state components, which |
409 | * are available to a thread group. The permission prctl() sets the |
410 | * enabled state bits in thread_group_leader()->thread.fpu. |
411 | * |
412 | * All run time operations use the per thread information in the |
413 | * currently active fpu.fpstate which contains the xfeature masks |
414 | * and sizes for kernel and user space. |
415 | * |
416 | * This master permission field is only to be used when |
417 | * task.fpu.fpstate based checks fail to validate whether the task |
418 | * is allowed to expand it's xfeatures set which requires to |
419 | * allocate a larger sized fpstate buffer. |
420 | * |
421 | * Do not access this field directly. Use the provided helper |
422 | * function. Unlocked access is possible for quick checks. |
423 | */ |
424 | u64 __state_perm; |
425 | |
426 | /* |
427 | * @__state_size: |
428 | * |
429 | * The size required for @__state_perm. Only valid to access |
430 | * with sighand locked. |
431 | */ |
432 | unsigned int __state_size; |
433 | |
434 | /* |
435 | * @__user_state_size: |
436 | * |
437 | * The size required for @__state_perm user part. Only valid to |
438 | * access with sighand locked. |
439 | */ |
440 | unsigned int __user_state_size; |
441 | }; |
442 | |
443 | /* |
444 | * Highest level per task FPU state data structure that |
445 | * contains the FPU register state plus various FPU |
446 | * state fields: |
447 | */ |
448 | struct fpu { |
449 | /* |
450 | * @last_cpu: |
451 | * |
452 | * Records the last CPU on which this context was loaded into |
453 | * FPU registers. (In the lazy-restore case we might be |
454 | * able to reuse FPU registers across multiple context switches |
455 | * this way, if no intermediate task used the FPU.) |
456 | * |
457 | * A value of -1 is used to indicate that the FPU state in context |
458 | * memory is newer than the FPU state in registers, and that the |
459 | * FPU state should be reloaded next time the task is run. |
460 | */ |
461 | unsigned int last_cpu; |
462 | |
463 | /* |
464 | * @avx512_timestamp: |
465 | * |
466 | * Records the timestamp of AVX512 use during last context switch. |
467 | */ |
468 | unsigned long avx512_timestamp; |
469 | |
470 | /* |
471 | * @fpstate: |
472 | * |
473 | * Pointer to the active struct fpstate. Initialized to |
474 | * point at @__fpstate below. |
475 | */ |
476 | struct fpstate *fpstate; |
477 | |
478 | /* |
479 | * @__task_fpstate: |
480 | * |
481 | * Pointer to an inactive struct fpstate. Initialized to NULL. Is |
482 | * used only for KVM support to swap out the regular task fpstate. |
483 | */ |
484 | struct fpstate *__task_fpstate; |
485 | |
486 | /* |
487 | * @perm: |
488 | * |
489 | * Permission related information |
490 | */ |
491 | struct fpu_state_perm perm; |
492 | |
493 | /* |
494 | * @guest_perm: |
495 | * |
496 | * Permission related information for guest pseudo FPUs |
497 | */ |
498 | struct fpu_state_perm guest_perm; |
499 | |
500 | /* |
501 | * @__fpstate: |
502 | * |
503 | * Initial in-memory storage for FPU registers which are saved in |
504 | * context switch and when the kernel uses the FPU. The registers |
505 | * are restored from this storage on return to user space if they |
506 | * are not longer containing the tasks FPU register state. |
507 | */ |
508 | struct fpstate __fpstate; |
509 | /* |
510 | * WARNING: '__fpstate' is dynamically-sized. Do not put |
511 | * anything after it here. |
512 | */ |
513 | }; |
514 | |
515 | /* |
516 | * Guest pseudo FPU container |
517 | */ |
518 | struct fpu_guest { |
519 | /* |
520 | * @xfeatures: xfeature bitmap of features which are |
521 | * currently enabled for the guest vCPU. |
522 | */ |
523 | u64 xfeatures; |
524 | |
525 | /* |
526 | * @perm: xfeature bitmap of features which are |
527 | * permitted to be enabled for the guest |
528 | * vCPU. |
529 | */ |
530 | u64 perm; |
531 | |
532 | /* |
533 | * @xfd_err: Save the guest value. |
534 | */ |
535 | u64 xfd_err; |
536 | |
537 | /* |
538 | * @uabi_size: Size required for save/restore |
539 | */ |
540 | unsigned int uabi_size; |
541 | |
542 | /* |
543 | * @fpstate: Pointer to the allocated guest fpstate |
544 | */ |
545 | struct fpstate *fpstate; |
546 | }; |
547 | |
548 | /* |
549 | * FPU state configuration data. Initialized at boot time. Read only after init. |
550 | */ |
551 | struct fpu_state_config { |
552 | /* |
553 | * @max_size: |
554 | * |
555 | * The maximum size of the register state buffer. Includes all |
556 | * supported features except independent managed features. |
557 | */ |
558 | unsigned int max_size; |
559 | |
560 | /* |
561 | * @default_size: |
562 | * |
563 | * The default size of the register state buffer. Includes all |
564 | * supported features except independent managed features and |
565 | * features which have to be requested by user space before usage. |
566 | */ |
567 | unsigned int default_size; |
568 | |
569 | /* |
570 | * @max_features: |
571 | * |
572 | * The maximum supported features bitmap. Does not include |
573 | * independent managed features. |
574 | */ |
575 | u64 max_features; |
576 | |
577 | /* |
578 | * @default_features: |
579 | * |
580 | * The default supported features bitmap. Does not include |
581 | * independent managed features and features which have to |
582 | * be requested by user space before usage. |
583 | */ |
584 | u64 default_features; |
585 | /* |
586 | * @legacy_features: |
587 | * |
588 | * Features which can be reported back to user space |
589 | * even without XSAVE support, i.e. legacy features FP + SSE |
590 | */ |
591 | u64 legacy_features; |
592 | }; |
593 | |
594 | /* FPU state configuration information */ |
595 | extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg; |
596 | |
597 | #endif /* _ASM_X86_FPU_H */ |
598 | |