1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * FPU data structures:
4 */
5#ifndef _ASM_X86_FPU_H
6#define _ASM_X86_FPU_H
7
8/*
9 * The legacy x87 FPU state format, as saved by FSAVE and
10 * restored by the FRSTOR instructions:
11 */
12struct fregs_state {
13 u32 cwd; /* FPU Control Word */
14 u32 swd; /* FPU Status Word */
15 u32 twd; /* FPU Tag Word */
16 u32 fip; /* FPU IP Offset */
17 u32 fcs; /* FPU IP Selector */
18 u32 foo; /* FPU Operand Pointer Offset */
19 u32 fos; /* FPU Operand Pointer Selector */
20
21 /* 8*10 bytes for each FP-reg = 80 bytes: */
22 u32 st_space[20];
23
24 /* Software status information [not touched by FSAVE]: */
25 u32 status;
26};
27
28/*
29 * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and
30 * restored by the FXRSTOR instructions. It's similar to the FSAVE
31 * format, but differs in some areas, plus has extensions at
32 * the end for the XMM registers.
33 */
34struct fxregs_state {
35 u16 cwd; /* Control Word */
36 u16 swd; /* Status Word */
37 u16 twd; /* Tag Word */
38 u16 fop; /* Last Instruction Opcode */
39 union {
40 struct {
41 u64 rip; /* Instruction Pointer */
42 u64 rdp; /* Data Pointer */
43 };
44 struct {
45 u32 fip; /* FPU IP Offset */
46 u32 fcs; /* FPU IP Selector */
47 u32 foo; /* FPU Operand Offset */
48 u32 fos; /* FPU Operand Selector */
49 };
50 };
51 u32 mxcsr; /* MXCSR Register State */
52 u32 mxcsr_mask; /* MXCSR Mask */
53
54 /* 8*16 bytes for each FP-reg = 128 bytes: */
55 u32 st_space[32];
56
57 /* 16*16 bytes for each XMM-reg = 256 bytes: */
58 u32 xmm_space[64];
59
60 u32 padding[12];
61
62 union {
63 u32 padding1[12];
64 u32 sw_reserved[12];
65 };
66
67} __attribute__((aligned(16)));
68
69/* Default value for fxregs_state.mxcsr: */
70#define MXCSR_DEFAULT 0x1f80
71
72/* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */
73#define MXCSR_AND_FLAGS_SIZE sizeof(u64)
74
75/*
76 * Software based FPU emulation state. This is arbitrary really,
77 * it matches the x87 format to make it easier to understand:
78 */
79struct swregs_state {
80 u32 cwd;
81 u32 swd;
82 u32 twd;
83 u32 fip;
84 u32 fcs;
85 u32 foo;
86 u32 fos;
87 /* 8*10 bytes for each FP-reg = 80 bytes: */
88 u32 st_space[20];
89 u8 ftop;
90 u8 changed;
91 u8 lookahead;
92 u8 no_update;
93 u8 rm;
94 u8 alimit;
95 struct math_emu_info *info;
96 u32 entry_eip;
97};
98
99/*
100 * List of XSAVE features Linux knows about:
101 */
102enum xfeature {
103 XFEATURE_FP,
104 XFEATURE_SSE,
105 /*
106 * Values above here are "legacy states".
107 * Those below are "extended states".
108 */
109 XFEATURE_YMM,
110 XFEATURE_BNDREGS,
111 XFEATURE_BNDCSR,
112 XFEATURE_OPMASK,
113 XFEATURE_ZMM_Hi256,
114 XFEATURE_Hi16_ZMM,
115 XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
116 XFEATURE_PKRU,
117 XFEATURE_PASID,
118 XFEATURE_CET_USER,
119 XFEATURE_CET_KERNEL_UNUSED,
120 XFEATURE_RSRVD_COMP_13,
121 XFEATURE_RSRVD_COMP_14,
122 XFEATURE_LBR,
123 XFEATURE_RSRVD_COMP_16,
124 XFEATURE_XTILE_CFG,
125 XFEATURE_XTILE_DATA,
126
127 XFEATURE_MAX,
128};
129
130#define XFEATURE_MASK_FP (1 << XFEATURE_FP)
131#define XFEATURE_MASK_SSE (1 << XFEATURE_SSE)
132#define XFEATURE_MASK_YMM (1 << XFEATURE_YMM)
133#define XFEATURE_MASK_BNDREGS (1 << XFEATURE_BNDREGS)
134#define XFEATURE_MASK_BNDCSR (1 << XFEATURE_BNDCSR)
135#define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK)
136#define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256)
137#define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM)
138#define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
139#define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU)
140#define XFEATURE_MASK_PASID (1 << XFEATURE_PASID)
141#define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER)
142#define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL_UNUSED)
143#define XFEATURE_MASK_LBR (1 << XFEATURE_LBR)
144#define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG)
145#define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA)
146
147#define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
148#define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \
149 | XFEATURE_MASK_ZMM_Hi256 \
150 | XFEATURE_MASK_Hi16_ZMM)
151
152#ifdef CONFIG_X86_64
153# define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA \
154 | XFEATURE_MASK_XTILE_CFG)
155#else
156# define XFEATURE_MASK_XTILE (0)
157#endif
158
159#define FIRST_EXTENDED_XFEATURE XFEATURE_YMM
160
161struct reg_128_bit {
162 u8 regbytes[128/8];
163};
164struct reg_256_bit {
165 u8 regbytes[256/8];
166};
167struct reg_512_bit {
168 u8 regbytes[512/8];
169};
170struct reg_1024_byte {
171 u8 regbytes[1024];
172};
173
174/*
175 * State component 2:
176 *
177 * There are 16x 256-bit AVX registers named YMM0-YMM15.
178 * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15)
179 * and are stored in 'struct fxregs_state::xmm_space[]' in the
180 * "legacy" area.
181 *
182 * The high 128 bits are stored here.
183 */
184struct ymmh_struct {
185 struct reg_128_bit hi_ymm[16];
186} __packed;
187
188/* Intel MPX support: */
189
190struct mpx_bndreg {
191 u64 lower_bound;
192 u64 upper_bound;
193} __packed;
194/*
195 * State component 3 is used for the 4 128-bit bounds registers
196 */
197struct mpx_bndreg_state {
198 struct mpx_bndreg bndreg[4];
199} __packed;
200
201/*
202 * State component 4 is used for the 64-bit user-mode MPX
203 * configuration register BNDCFGU and the 64-bit MPX status
204 * register BNDSTATUS. We call the pair "BNDCSR".
205 */
206struct mpx_bndcsr {
207 u64 bndcfgu;
208 u64 bndstatus;
209} __packed;
210
211/*
212 * The BNDCSR state is padded out to be 64-bytes in size.
213 */
214struct mpx_bndcsr_state {
215 union {
216 struct mpx_bndcsr bndcsr;
217 u8 pad_to_64_bytes[64];
218 };
219} __packed;
220
221/* AVX-512 Components: */
222
223/*
224 * State component 5 is used for the 8 64-bit opmask registers
225 * k0-k7 (opmask state).
226 */
227struct avx_512_opmask_state {
228 u64 opmask_reg[8];
229} __packed;
230
231/*
232 * State component 6 is used for the upper 256 bits of the
233 * registers ZMM0-ZMM15. These 16 256-bit values are denoted
234 * ZMM0_H-ZMM15_H (ZMM_Hi256 state).
235 */
236struct avx_512_zmm_uppers_state {
237 struct reg_256_bit zmm_upper[16];
238} __packed;
239
240/*
241 * State component 7 is used for the 16 512-bit registers
242 * ZMM16-ZMM31 (Hi16_ZMM state).
243 */
244struct avx_512_hi16_state {
245 struct reg_512_bit hi16_zmm[16];
246} __packed;
247
248/*
249 * State component 9: 32-bit PKRU register. The state is
250 * 8 bytes long but only 4 bytes is used currently.
251 */
252struct pkru_state {
253 u32 pkru;
254 u32 pad;
255} __packed;
256
257/*
258 * State component 11 is Control-flow Enforcement user states
259 */
260struct cet_user_state {
261 /* user control-flow settings */
262 u64 user_cet;
263 /* user shadow stack pointer */
264 u64 user_ssp;
265};
266
267/*
268 * State component 15: Architectural LBR configuration state.
269 * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
270 */
271
272struct lbr_entry {
273 u64 from;
274 u64 to;
275 u64 info;
276};
277
278struct arch_lbr_state {
279 u64 lbr_ctl;
280 u64 lbr_depth;
281 u64 ler_from;
282 u64 ler_to;
283 u64 ler_info;
284 struct lbr_entry entries[];
285};
286
287/*
288 * State component 17: 64-byte tile configuration register.
289 */
290struct xtile_cfg {
291 u64 tcfg[8];
292} __packed;
293
294/*
295 * State component 18: 1KB tile data register.
296 * Each register represents 16 64-byte rows of the matrix
297 * data. But the number of registers depends on the actual
298 * implementation.
299 */
300struct xtile_data {
301 struct reg_1024_byte tmm;
302} __packed;
303
304/*
305 * State component 10 is supervisor state used for context-switching the
306 * PASID state.
307 */
308struct ia32_pasid_state {
309 u64 pasid;
310} __packed;
311
312struct xstate_header {
313 u64 xfeatures;
314 u64 xcomp_bv;
315 u64 reserved[6];
316} __attribute__((packed));
317
318/*
319 * xstate_header.xcomp_bv[63] indicates that the extended_state_area
320 * is in compacted format.
321 */
322#define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63)
323
324/*
325 * This is our most modern FPU state format, as saved by the XSAVE
326 * and restored by the XRSTOR instructions.
327 *
328 * It consists of a legacy fxregs portion, an xstate header and
329 * subsequent areas as defined by the xstate header. Not all CPUs
330 * support all the extensions, so the size of the extended area
331 * can vary quite a bit between CPUs.
332 */
333struct xregs_state {
334 struct fxregs_state i387;
335 struct xstate_header header;
336 u8 extended_state_area[];
337} __attribute__ ((packed, aligned (64)));
338
339/*
340 * This is a union of all the possible FPU state formats
341 * put together, so that we can pick the right one runtime.
342 *
343 * The size of the structure is determined by the largest
344 * member - which is the xsave area. The padding is there
345 * to ensure that statically-allocated task_structs (just
346 * the init_task today) have enough space.
347 */
348union fpregs_state {
349 struct fregs_state fsave;
350 struct fxregs_state fxsave;
351 struct swregs_state soft;
352 struct xregs_state xsave;
353 u8 __padding[PAGE_SIZE];
354};
355
356struct fpstate {
357 /* @kernel_size: The size of the kernel register image */
358 unsigned int size;
359
360 /* @user_size: The size in non-compacted UABI format */
361 unsigned int user_size;
362
363 /* @xfeatures: xfeatures for which the storage is sized */
364 u64 xfeatures;
365
366 /* @user_xfeatures: xfeatures valid in UABI buffers */
367 u64 user_xfeatures;
368
369 /* @xfd: xfeatures disabled to trap userspace use. */
370 u64 xfd;
371
372 /* @is_valloc: Indicator for dynamically allocated state */
373 unsigned int is_valloc : 1;
374
375 /* @is_guest: Indicator for guest state (KVM) */
376 unsigned int is_guest : 1;
377
378 /*
379 * @is_confidential: Indicator for KVM confidential mode.
380 * The FPU registers are restored by the
381 * vmentry firmware from encrypted guest
382 * memory. On vmexit the FPU registers are
383 * saved by firmware to encrypted guest memory
384 * and the registers are scrubbed before
385 * returning to the host. So there is no
386 * content which is worth saving and restoring.
387 * The fpstate has to be there so that
388 * preemption and softirq FPU usage works
389 * without special casing.
390 */
391 unsigned int is_confidential : 1;
392
393 /* @in_use: State is in use */
394 unsigned int in_use : 1;
395
396 /* @regs: The register state union for all supported formats */
397 union fpregs_state regs;
398
399 /* @regs is dynamically sized! Don't add anything after @regs! */
400} __aligned(64);
401
402#define FPU_GUEST_PERM_LOCKED BIT_ULL(63)
403
404struct fpu_state_perm {
405 /*
406 * @__state_perm:
407 *
408 * This bitmap indicates the permission for state components, which
409 * are available to a thread group. The permission prctl() sets the
410 * enabled state bits in thread_group_leader()->thread.fpu.
411 *
412 * All run time operations use the per thread information in the
413 * currently active fpu.fpstate which contains the xfeature masks
414 * and sizes for kernel and user space.
415 *
416 * This master permission field is only to be used when
417 * task.fpu.fpstate based checks fail to validate whether the task
418 * is allowed to expand it's xfeatures set which requires to
419 * allocate a larger sized fpstate buffer.
420 *
421 * Do not access this field directly. Use the provided helper
422 * function. Unlocked access is possible for quick checks.
423 */
424 u64 __state_perm;
425
426 /*
427 * @__state_size:
428 *
429 * The size required for @__state_perm. Only valid to access
430 * with sighand locked.
431 */
432 unsigned int __state_size;
433
434 /*
435 * @__user_state_size:
436 *
437 * The size required for @__state_perm user part. Only valid to
438 * access with sighand locked.
439 */
440 unsigned int __user_state_size;
441};
442
443/*
444 * Highest level per task FPU state data structure that
445 * contains the FPU register state plus various FPU
446 * state fields:
447 */
448struct fpu {
449 /*
450 * @last_cpu:
451 *
452 * Records the last CPU on which this context was loaded into
453 * FPU registers. (In the lazy-restore case we might be
454 * able to reuse FPU registers across multiple context switches
455 * this way, if no intermediate task used the FPU.)
456 *
457 * A value of -1 is used to indicate that the FPU state in context
458 * memory is newer than the FPU state in registers, and that the
459 * FPU state should be reloaded next time the task is run.
460 */
461 unsigned int last_cpu;
462
463 /*
464 * @avx512_timestamp:
465 *
466 * Records the timestamp of AVX512 use during last context switch.
467 */
468 unsigned long avx512_timestamp;
469
470 /*
471 * @fpstate:
472 *
473 * Pointer to the active struct fpstate. Initialized to
474 * point at @__fpstate below.
475 */
476 struct fpstate *fpstate;
477
478 /*
479 * @__task_fpstate:
480 *
481 * Pointer to an inactive struct fpstate. Initialized to NULL. Is
482 * used only for KVM support to swap out the regular task fpstate.
483 */
484 struct fpstate *__task_fpstate;
485
486 /*
487 * @perm:
488 *
489 * Permission related information
490 */
491 struct fpu_state_perm perm;
492
493 /*
494 * @guest_perm:
495 *
496 * Permission related information for guest pseudo FPUs
497 */
498 struct fpu_state_perm guest_perm;
499
500 /*
501 * @__fpstate:
502 *
503 * Initial in-memory storage for FPU registers which are saved in
504 * context switch and when the kernel uses the FPU. The registers
505 * are restored from this storage on return to user space if they
506 * are not longer containing the tasks FPU register state.
507 */
508 struct fpstate __fpstate;
509 /*
510 * WARNING: '__fpstate' is dynamically-sized. Do not put
511 * anything after it here.
512 */
513};
514
515/*
516 * Guest pseudo FPU container
517 */
518struct fpu_guest {
519 /*
520 * @xfeatures: xfeature bitmap of features which are
521 * currently enabled for the guest vCPU.
522 */
523 u64 xfeatures;
524
525 /*
526 * @perm: xfeature bitmap of features which are
527 * permitted to be enabled for the guest
528 * vCPU.
529 */
530 u64 perm;
531
532 /*
533 * @xfd_err: Save the guest value.
534 */
535 u64 xfd_err;
536
537 /*
538 * @uabi_size: Size required for save/restore
539 */
540 unsigned int uabi_size;
541
542 /*
543 * @fpstate: Pointer to the allocated guest fpstate
544 */
545 struct fpstate *fpstate;
546};
547
548/*
549 * FPU state configuration data. Initialized at boot time. Read only after init.
550 */
551struct fpu_state_config {
552 /*
553 * @max_size:
554 *
555 * The maximum size of the register state buffer. Includes all
556 * supported features except independent managed features.
557 */
558 unsigned int max_size;
559
560 /*
561 * @default_size:
562 *
563 * The default size of the register state buffer. Includes all
564 * supported features except independent managed features and
565 * features which have to be requested by user space before usage.
566 */
567 unsigned int default_size;
568
569 /*
570 * @max_features:
571 *
572 * The maximum supported features bitmap. Does not include
573 * independent managed features.
574 */
575 u64 max_features;
576
577 /*
578 * @default_features:
579 *
580 * The default supported features bitmap. Does not include
581 * independent managed features and features which have to
582 * be requested by user space before usage.
583 */
584 u64 default_features;
585 /*
586 * @legacy_features:
587 *
588 * Features which can be reported back to user space
589 * even without XSAVE support, i.e. legacy features FP + SSE
590 */
591 u64 legacy_features;
592};
593
594/* FPU state configuration information */
595extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg;
596
597#endif /* _ASM_X86_FPU_H */
598

source code of linux/arch/x86/include/asm/fpu/types.h