1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * linux/arch/x86_64/entry.S
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
7 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
8 *
9 * entry.S contains the system-call and fault low-level handling routines.
10 *
11 * Some of this is documented in Documentation/arch/x86/entry_64.rst
12 *
13 * A note on terminology:
14 * - iret frame: Architecture defined interrupt frame from SS to RIP
15 * at the top of the kernel process stack.
16 *
17 * Some macro usage:
18 * - SYM_FUNC_START/END:Define functions in the symbol table.
19 * - idtentry: Define exception entry points.
20 */
21#include <linux/export.h>
22#include <linux/kvm_types.h>
23#include <linux/linkage.h>
24#include <asm/segment.h>
25#include <asm/cache.h>
26#include <asm/errno.h>
27#include <asm/asm-offsets.h>
28#include <asm/msr.h>
29#include <asm/unistd.h>
30#include <asm/thread_info.h>
31#include <asm/hw_irq.h>
32#include <asm/page_types.h>
33#include <asm/irqflags.h>
34#include <asm/paravirt.h>
35#include <asm/percpu.h>
36#include <asm/asm.h>
37#include <asm/smap.h>
38#include <asm/pgtable_types.h>
39#include <asm/frame.h>
40#include <asm/trapnr.h>
41#include <asm/nospec-branch.h>
42#include <asm/fsgsbase.h>
43#include <linux/err.h>
44
45#include "calling.h"
46
47.code64
48.section .entry.text, "ax"
49
50/*
51 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
52 *
53 * This is the only entry point used for 64-bit system calls. The
54 * hardware interface is reasonably well designed and the register to
55 * argument mapping Linux uses fits well with the registers that are
56 * available when SYSCALL is used.
57 *
58 * SYSCALL instructions can be found inlined in libc implementations as
59 * well as some other programs and libraries. There are also a handful
60 * of SYSCALL instructions in the vDSO used, for example, as a
61 * clock_gettimeofday fallback.
62 *
63 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
64 * then loads new ss, cs, and rip from previously programmed MSRs.
65 * rflags gets masked by a value from another MSR (so CLD and CLAC
66 * are not needed). SYSCALL does not save anything on the stack
67 * and does not change rsp.
68 *
69 * Registers on entry:
70 * rax system call number
71 * rcx return address
72 * r11 saved rflags (note: r11 is callee-clobbered register in C ABI)
73 * rdi arg0
74 * rsi arg1
75 * rdx arg2
76 * r10 arg3 (needs to be moved to rcx to conform to C ABI)
77 * r8 arg4
78 * r9 arg5
79 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
80 *
81 * Only called from user space.
82 *
83 * When user can change pt_regs->foo always force IRET. That is because
84 * it deals with uncanonical addresses better. SYSRET has trouble
85 * with them due to bugs in both AMD and Intel CPUs.
86 */
87
88SYM_CODE_START(entry_SYSCALL_64)
89 UNWIND_HINT_ENTRY
90 ENDBR
91
92 swapgs
93 /* tss.sp2 is scratch space. */
94 movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
95 SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
96 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
97
98SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
99 ANNOTATE_NOENDBR
100
101 /* Construct struct pt_regs on stack */
102 pushq $__USER_DS /* pt_regs->ss */
103 pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) /* pt_regs->sp */
104 pushq %r11 /* pt_regs->flags */
105 pushq $__USER_CS /* pt_regs->cs */
106 pushq %rcx /* pt_regs->ip */
107SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL)
108 pushq %rax /* pt_regs->orig_ax */
109
110 PUSH_AND_CLEAR_REGS rax=$-ENOSYS
111
112 /* IRQs are off. */
113 movq %rsp, %rdi
114 /* Sign extend the lower 32bit as syscall numbers are treated as int */
115 movslq %eax, %rsi
116
117 /* clobbers %rax, make sure it is after saving the syscall nr */
118 IBRS_ENTER
119 UNTRAIN_RET
120 CLEAR_BRANCH_HISTORY
121
122 call do_syscall_64 /* returns with IRQs disabled */
123
124 /*
125 * Try to use SYSRET instead of IRET if we're returning to
126 * a completely clean 64-bit userspace context. If we're not,
127 * go to the slow exit path.
128 * In the Xen PV case we must use iret anyway.
129 */
130
131 ALTERNATIVE "testb %al, %al; jz swapgs_restore_regs_and_return_to_usermode", \
132 "jmp swapgs_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
133
134 /*
135 * We win! This label is here just for ease of understanding
136 * perf profiles. Nothing jumps here.
137 */
138syscall_return_via_sysret:
139 IBRS_EXIT
140 POP_REGS pop_rdi=0
141
142 /*
143 * Now all regs are restored except RSP and RDI.
144 * Save old stack pointer and switch to trampoline stack.
145 */
146 movq %rsp, %rdi
147 movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
148 UNWIND_HINT_END_OF_STACK
149
150 pushq RSP-RDI(%rdi) /* RSP */
151 pushq (%rdi) /* RDI */
152
153 /*
154 * We are on the trampoline stack. All regs except RDI are live.
155 * We can do future final exit work right here.
156 */
157 STACKLEAK_ERASE_NOCLOBBER
158
159 SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
160
161 popq %rdi
162 popq %rsp
163SYM_INNER_LABEL(entry_SYSRETQ_unsafe_stack, SYM_L_GLOBAL)
164 ANNOTATE_NOENDBR
165 swapgs
166 CLEAR_CPU_BUFFERS
167 sysretq
168SYM_INNER_LABEL(entry_SYSRETQ_end, SYM_L_GLOBAL)
169 ANNOTATE_NOENDBR
170 int3
171SYM_CODE_END(entry_SYSCALL_64)
172
173/*
174 * %rdi: prev task
175 * %rsi: next task
176 */
177.pushsection .text, "ax"
178SYM_FUNC_START(__switch_to_asm)
179 ANNOTATE_NOENDBR
180 /*
181 * Save callee-saved registers
182 * This must match the order in inactive_task_frame
183 */
184 pushq %rbp
185 pushq %rbx
186 pushq %r12
187 pushq %r13
188 pushq %r14
189 pushq %r15
190
191 /* switch stack */
192 movq %rsp, TASK_threadsp(%rdi)
193 movq TASK_threadsp(%rsi), %rsp
194
195#ifdef CONFIG_STACKPROTECTOR
196 movq TASK_stack_canary(%rsi), %rbx
197 movq %rbx, PER_CPU_VAR(__stack_chk_guard)
198#endif
199
200 /*
201 * When switching from a shallower to a deeper call stack
202 * the RSB may either underflow or use entries populated
203 * with userspace addresses. On CPUs where those concerns
204 * exist, overwrite the RSB with entries which capture
205 * speculative execution to prevent attack.
206 */
207 FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
208
209 /* restore callee-saved registers */
210 popq %r15
211 popq %r14
212 popq %r13
213 popq %r12
214 popq %rbx
215 popq %rbp
216
217 jmp __switch_to
218SYM_FUNC_END(__switch_to_asm)
219.popsection
220
221/*
222 * A newly forked process directly context switches into this address.
223 *
224 * rax: prev task we switched from
225 * rbx: kernel thread func (NULL for user thread)
226 * r12: kernel thread arg
227 */
228.pushsection .text, "ax"
229SYM_CODE_START(ret_from_fork_asm)
230 /*
231 * This is the start of the kernel stack; even through there's a
232 * register set at the top, the regset isn't necessarily coherent
233 * (consider kthreads) and one cannot unwind further.
234 *
235 * This ensures stack unwinds of kernel threads terminate in a known
236 * good state.
237 */
238 UNWIND_HINT_END_OF_STACK
239 ANNOTATE_NOENDBR // copy_thread
240 CALL_DEPTH_ACCOUNT
241
242 movq %rax, %rdi /* prev */
243 movq %rsp, %rsi /* regs */
244 movq %rbx, %rdx /* fn */
245 movq %r12, %rcx /* fn_arg */
246 call ret_from_fork
247
248 /*
249 * Set the stack state to what is expected for the target function
250 * -- at this point the register set should be a valid user set
251 * and unwind should work normally.
252 */
253 UNWIND_HINT_REGS
254
255#ifdef CONFIG_X86_FRED
256 ALTERNATIVE "jmp swapgs_restore_regs_and_return_to_usermode", \
257 "jmp asm_fred_exit_user", X86_FEATURE_FRED
258#else
259 jmp swapgs_restore_regs_and_return_to_usermode
260#endif
261SYM_CODE_END(ret_from_fork_asm)
262.popsection
263
264.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
265#ifdef CONFIG_DEBUG_ENTRY
266 pushq %rax
267 SAVE_FLAGS
268 testl $X86_EFLAGS_IF, %eax
269 jz .Lokay_\@
270 ud2
271.Lokay_\@:
272 popq %rax
273#endif
274.endm
275
276SYM_CODE_START(xen_error_entry)
277 ANNOTATE_NOENDBR
278 UNWIND_HINT_FUNC
279 PUSH_AND_CLEAR_REGS save_ret=1
280 ENCODE_FRAME_POINTER 8
281 UNTRAIN_RET_FROM_CALL
282 RET
283SYM_CODE_END(xen_error_entry)
284
285/**
286 * idtentry_body - Macro to emit code calling the C function
287 * @cfunc: C function to be called
288 * @has_error_code: Hardware pushed error code on stack
289 */
290.macro idtentry_body cfunc has_error_code:req
291
292 /*
293 * Call error_entry() and switch to the task stack if from userspace.
294 *
295 * When in XENPV, it is already in the task stack, and it can't fault
296 * for native_iret() nor native_load_gs_index() since XENPV uses its
297 * own pvops for IRET and load_gs_index(). And it doesn't need to
298 * switch the CR3. So it can skip invoking error_entry().
299 */
300 ALTERNATIVE "call error_entry; movq %rax, %rsp", \
301 "call xen_error_entry", X86_FEATURE_XENPV
302
303 ENCODE_FRAME_POINTER
304 UNWIND_HINT_REGS
305
306 movq %rsp, %rdi /* pt_regs pointer into 1st argument*/
307
308 .if \has_error_code == 1
309 movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
310 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
311 .endif
312
313 /* For some configurations \cfunc ends up being a noreturn. */
314 ANNOTATE_REACHABLE
315 call \cfunc
316
317 jmp error_return
318.endm
319
320/**
321 * idtentry - Macro to generate entry stubs for simple IDT entries
322 * @vector: Vector number
323 * @asmsym: ASM symbol for the entry point
324 * @cfunc: C function to be called
325 * @has_error_code: Hardware pushed error code on stack
326 *
327 * The macro emits code to set up the kernel context for straight forward
328 * and simple IDT entries. No IST stack, no paranoid entry checks.
329 */
330.macro idtentry vector asmsym cfunc has_error_code:req
331SYM_CODE_START(\asmsym)
332
333 .if \vector == X86_TRAP_BP
334 /* #BP advances %rip to the next instruction */
335 UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8 signal=0
336 .else
337 UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8
338 .endif
339
340 ENDBR
341 ASM_CLAC
342 cld
343
344 .if \has_error_code == 0
345 pushq $-1 /* ORIG_RAX: no syscall to restart */
346 .endif
347
348 .if \vector == X86_TRAP_BP
349 /*
350 * If coming from kernel space, create a 6-word gap to allow the
351 * int3 handler to emulate a call instruction.
352 */
353 testb $3, CS-ORIG_RAX(%rsp)
354 jnz .Lfrom_usermode_no_gap_\@
355 .rept 6
356 pushq 5*8(%rsp)
357 .endr
358 UNWIND_HINT_IRET_REGS offset=8
359.Lfrom_usermode_no_gap_\@:
360 .endif
361
362 idtentry_body \cfunc \has_error_code
363
364_ASM_NOKPROBE(\asmsym)
365SYM_CODE_END(\asmsym)
366.endm
367
368/*
369 * Interrupt entry/exit.
370 *
371 + The interrupt stubs push (vector) onto the stack, which is the error_code
372 * position of idtentry exceptions, and jump to one of the two idtentry points
373 * (common/spurious).
374 *
375 * common_interrupt is a hotpath, align it to a cache line
376 */
377.macro idtentry_irq vector cfunc
378 .p2align CONFIG_X86_L1_CACHE_SHIFT
379 idtentry \vector asm_\cfunc \cfunc has_error_code=1
380.endm
381
382/**
383 * idtentry_mce_db - Macro to generate entry stubs for #MC and #DB
384 * @vector: Vector number
385 * @asmsym: ASM symbol for the entry point
386 * @cfunc: C function to be called
387 *
388 * The macro emits code to set up the kernel context for #MC and #DB
389 *
390 * If the entry comes from user space it uses the normal entry path
391 * including the return to user space work and preemption checks on
392 * exit.
393 *
394 * If hits in kernel mode then it needs to go through the paranoid
395 * entry as the exception can hit any random state. No preemption
396 * check on exit to keep the paranoid path simple.
397 */
398.macro idtentry_mce_db vector asmsym cfunc
399SYM_CODE_START(\asmsym)
400 UNWIND_HINT_IRET_ENTRY
401 ENDBR
402 ASM_CLAC
403 cld
404
405 pushq $-1 /* ORIG_RAX: no syscall to restart */
406
407 /*
408 * If the entry is from userspace, switch stacks and treat it as
409 * a normal entry.
410 */
411 testb $3, CS-ORIG_RAX(%rsp)
412 jnz .Lfrom_usermode_switch_stack_\@
413
414 /* paranoid_entry returns GS information for paranoid_exit in EBX. */
415 call paranoid_entry
416
417 UNWIND_HINT_REGS
418
419 movq %rsp, %rdi /* pt_regs pointer */
420
421 call \cfunc
422
423 jmp paranoid_exit
424
425 /* Switch to the regular task stack and use the noist entry point */
426.Lfrom_usermode_switch_stack_\@:
427 idtentry_body noist_\cfunc, has_error_code=0
428
429_ASM_NOKPROBE(\asmsym)
430SYM_CODE_END(\asmsym)
431.endm
432
433#ifdef CONFIG_AMD_MEM_ENCRYPT
434/**
435 * idtentry_vc - Macro to generate entry stub for #VC
436 * @vector: Vector number
437 * @asmsym: ASM symbol for the entry point
438 * @cfunc: C function to be called
439 *
440 * The macro emits code to set up the kernel context for #VC. The #VC handler
441 * runs on an IST stack and needs to be able to cause nested #VC exceptions.
442 *
443 * To make this work the #VC entry code tries its best to pretend it doesn't use
444 * an IST stack by switching to the task stack if coming from user-space (which
445 * includes early SYSCALL entry path) or back to the stack in the IRET frame if
446 * entered from kernel-mode.
447 *
448 * If entered from kernel-mode the return stack is validated first, and if it is
449 * not safe to use (e.g. because it points to the entry stack) the #VC handler
450 * will switch to a fall-back stack (VC2) and call a special handler function.
451 *
452 * The macro is only used for one vector, but it is planned to be extended in
453 * the future for the #HV exception.
454 */
455.macro idtentry_vc vector asmsym cfunc
456SYM_CODE_START(\asmsym)
457 UNWIND_HINT_IRET_ENTRY
458 ENDBR
459 ASM_CLAC
460 cld
461
462 /*
463 * If the entry is from userspace, switch stacks and treat it as
464 * a normal entry.
465 */
466 testb $3, CS-ORIG_RAX(%rsp)
467 jnz .Lfrom_usermode_switch_stack_\@
468
469 /*
470 * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX.
471 * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS
472 */
473 call paranoid_entry
474
475 UNWIND_HINT_REGS
476
477 /*
478 * Switch off the IST stack to make it free for nested exceptions. The
479 * vc_switch_off_ist() function will switch back to the interrupted
480 * stack if it is safe to do so. If not it switches to the VC fall-back
481 * stack.
482 */
483 movq %rsp, %rdi /* pt_regs pointer */
484 call vc_switch_off_ist
485 movq %rax, %rsp /* Switch to new stack */
486
487 ENCODE_FRAME_POINTER
488 UNWIND_HINT_REGS
489
490 /* Update pt_regs */
491 movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
492 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
493
494 movq %rsp, %rdi /* pt_regs pointer */
495
496 call kernel_\cfunc
497
498 /*
499 * No need to switch back to the IST stack. The current stack is either
500 * identical to the stack in the IRET frame or the VC fall-back stack,
501 * so it is definitely mapped even with PTI enabled.
502 */
503 jmp paranoid_exit
504
505 /* Switch to the regular task stack */
506.Lfrom_usermode_switch_stack_\@:
507 idtentry_body user_\cfunc, has_error_code=1
508
509_ASM_NOKPROBE(\asmsym)
510SYM_CODE_END(\asmsym)
511.endm
512#endif
513
514/*
515 * Double fault entry. Straight paranoid. No checks from which context
516 * this comes because for the espfix induced #DF this would do the wrong
517 * thing.
518 */
519.macro idtentry_df vector asmsym cfunc
520SYM_CODE_START(\asmsym)
521 UNWIND_HINT_IRET_ENTRY offset=8
522 ENDBR
523 ASM_CLAC
524 cld
525
526 /* paranoid_entry returns GS information for paranoid_exit in EBX. */
527 call paranoid_entry
528 UNWIND_HINT_REGS
529
530 movq %rsp, %rdi /* pt_regs pointer into first argument */
531 movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
532 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
533
534 /* For some configurations \cfunc ends up being a noreturn. */
535 ANNOTATE_REACHABLE
536 call \cfunc
537
538 jmp paranoid_exit
539
540_ASM_NOKPROBE(\asmsym)
541SYM_CODE_END(\asmsym)
542.endm
543
544/*
545 * Include the defines which emit the idt entries which are shared
546 * shared between 32 and 64 bit and emit the __irqentry_text_* markers
547 * so the stacktrace boundary checks work.
548 */
549 __ALIGN
550 .globl __irqentry_text_start
551__irqentry_text_start:
552
553#include <asm/idtentry.h>
554
555 __ALIGN
556 .globl __irqentry_text_end
557__irqentry_text_end:
558 ANNOTATE_NOENDBR
559
560SYM_CODE_START_LOCAL(common_interrupt_return)
561SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
562 IBRS_EXIT
563#ifdef CONFIG_XEN_PV
564 ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
565#endif
566#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
567 ALTERNATIVE "", "jmp .Lpti_restore_regs_and_return_to_usermode", X86_FEATURE_PTI
568#endif
569
570 STACKLEAK_ERASE
571 POP_REGS
572 add $8, %rsp /* orig_ax */
573 UNWIND_HINT_IRET_REGS
574
575.Lswapgs_and_iret:
576 swapgs
577 CLEAR_CPU_BUFFERS
578 /* Assert that the IRET frame indicates user mode. */
579 testb $3, 8(%rsp)
580 jnz .Lnative_iret
581 ud2
582
583#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
584.Lpti_restore_regs_and_return_to_usermode:
585 POP_REGS pop_rdi=0
586
587 /*
588 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
589 * Save old stack pointer and switch to trampoline stack.
590 */
591 movq %rsp, %rdi
592 movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
593 UNWIND_HINT_END_OF_STACK
594
595 /* Copy the IRET frame to the trampoline stack. */
596 pushq 6*8(%rdi) /* SS */
597 pushq 5*8(%rdi) /* RSP */
598 pushq 4*8(%rdi) /* EFLAGS */
599 pushq 3*8(%rdi) /* CS */
600 pushq 2*8(%rdi) /* RIP */
601
602 /* Push user RDI on the trampoline stack. */
603 pushq (%rdi)
604
605 /*
606 * We are on the trampoline stack. All regs except RDI are live.
607 * We can do future final exit work right here.
608 */
609 STACKLEAK_ERASE_NOCLOBBER
610
611 push %rax
612 SWITCH_TO_USER_CR3 scratch_reg=%rdi scratch_reg2=%rax
613 pop %rax
614
615 /* Restore RDI. */
616 popq %rdi
617 jmp .Lswapgs_and_iret
618#endif
619
620SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
621#ifdef CONFIG_DEBUG_ENTRY
622 /* Assert that pt_regs indicates kernel mode. */
623 testb $3, CS(%rsp)
624 jz 1f
625 ud2
6261:
627#endif
628 POP_REGS
629 addq $8, %rsp /* skip regs->orig_ax */
630 /*
631 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
632 * when returning from IPI handler.
633 */
634#ifdef CONFIG_XEN_PV
635SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL)
636 ANNOTATE_NOENDBR
637 .byte 0xe9
638 .long .Lnative_iret - (. + 4)
639#endif
640
641.Lnative_iret:
642 UNWIND_HINT_IRET_REGS
643 /*
644 * Are we returning to a stack segment from the LDT? Note: in
645 * 64-bit mode SS:RSP on the exception stack is always valid.
646 */
647#ifdef CONFIG_X86_ESPFIX64
648 testb $4, (SS-RIP)(%rsp)
649 jnz native_irq_return_ldt
650#endif
651
652SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL)
653 ANNOTATE_NOENDBR // exc_double_fault
654 /*
655 * This may fault. Non-paranoid faults on return to userspace are
656 * handled by fixup_bad_iret. These include #SS, #GP, and #NP.
657 * Double-faults due to espfix64 are handled in exc_double_fault.
658 * Other faults here are fatal.
659 */
660 iretq
661
662#ifdef CONFIG_X86_ESPFIX64
663native_irq_return_ldt:
664 /*
665 * We are running with user GSBASE. All GPRs contain their user
666 * values. We have a percpu ESPFIX stack that is eight slots
667 * long (see ESPFIX_STACK_SIZE). espfix_waddr points to the bottom
668 * of the ESPFIX stack.
669 *
670 * We clobber RAX and RDI in this code. We stash RDI on the
671 * normal stack and RAX on the ESPFIX stack.
672 *
673 * The ESPFIX stack layout we set up looks like this:
674 *
675 * --- top of ESPFIX stack ---
676 * SS
677 * RSP
678 * RFLAGS
679 * CS
680 * RIP <-- RSP points here when we're done
681 * RAX <-- espfix_waddr points here
682 * --- bottom of ESPFIX stack ---
683 */
684
685 pushq %rdi /* Stash user RDI */
686 swapgs /* to kernel GS */
687 SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */
688
689 movq PER_CPU_VAR(espfix_waddr), %rdi
690 movq %rax, (0*8)(%rdi) /* user RAX */
691 movq (1*8)(%rsp), %rax /* user RIP */
692 movq %rax, (1*8)(%rdi)
693 movq (2*8)(%rsp), %rax /* user CS */
694 movq %rax, (2*8)(%rdi)
695 movq (3*8)(%rsp), %rax /* user RFLAGS */
696 movq %rax, (3*8)(%rdi)
697 movq (5*8)(%rsp), %rax /* user SS */
698 movq %rax, (5*8)(%rdi)
699 movq (4*8)(%rsp), %rax /* user RSP */
700 movq %rax, (4*8)(%rdi)
701 /* Now RAX == RSP. */
702
703 andl $0xffff0000, %eax /* RAX = (RSP & 0xffff0000) */
704
705 /*
706 * espfix_stack[31:16] == 0. The page tables are set up such that
707 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
708 * espfix_waddr for any X. That is, there are 65536 RO aliases of
709 * the same page. Set up RSP so that RSP[31:16] contains the
710 * respective 16 bits of the /userspace/ RSP and RSP nonetheless
711 * still points to an RO alias of the ESPFIX stack.
712 */
713 orq PER_CPU_VAR(espfix_stack), %rax
714
715 SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
716 swapgs /* to user GS */
717 popq %rdi /* Restore user RDI */
718
719 movq %rax, %rsp
720 UNWIND_HINT_IRET_REGS offset=8
721
722 /*
723 * At this point, we cannot write to the stack any more, but we can
724 * still read.
725 */
726 popq %rax /* Restore user RAX */
727
728 CLEAR_CPU_BUFFERS
729
730 /*
731 * RSP now points to an ordinary IRET frame, except that the page
732 * is read-only and RSP[31:16] are preloaded with the userspace
733 * values. We can now IRET back to userspace.
734 */
735 jmp native_irq_return_iret
736#endif
737SYM_CODE_END(common_interrupt_return)
738_ASM_NOKPROBE(common_interrupt_return)
739
740/*
741 * Reload gs selector with exception handling
742 * di: new selector
743 *
744 * Is in entry.text as it shouldn't be instrumented.
745 */
746SYM_FUNC_START(asm_load_gs_index)
747 ANNOTATE_NOENDBR
748 FRAME_BEGIN
749 swapgs
750.Lgs_change:
751 ANNOTATE_NOENDBR // error_entry
752 movl %edi, %gs
7532: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
754 swapgs
755 FRAME_END
756 RET
757
758 /* running with kernelgs */
759.Lbad_gs:
760 swapgs /* switch back to user gs */
761.macro ZAP_GS
762 /* This can't be a string because the preprocessor needs to see it. */
763 movl $__USER_DS, %eax
764 movl %eax, %gs
765.endm
766 ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
767 xorl %eax, %eax
768 movl %eax, %gs
769 jmp 2b
770
771 _ASM_EXTABLE(.Lgs_change, .Lbad_gs)
772
773SYM_FUNC_END(asm_load_gs_index)
774EXPORT_SYMBOL(asm_load_gs_index)
775
776#ifdef CONFIG_XEN_PV
777/*
778 * A note on the "critical region" in our callback handler.
779 * We want to avoid stacking callback handlers due to events occurring
780 * during handling of the last event. To do this, we keep events disabled
781 * until we've done all processing. HOWEVER, we must enable events before
782 * popping the stack frame (can't be done atomically) and so it would still
783 * be possible to get enough handler activations to overflow the stack.
784 * Although unlikely, bugs of that kind are hard to track down, so we'd
785 * like to avoid the possibility.
786 * So, on entry to the handler we detect whether we interrupted an
787 * existing activation in its critical region -- if so, we pop the current
788 * activation and restart the handler using the previous one.
789 *
790 * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs)
791 */
792 __FUNC_ALIGN
793SYM_CODE_START_LOCAL_NOALIGN(exc_xen_hypervisor_callback)
794
795/*
796 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
797 * see the correct pointer to the pt_regs
798 */
799 UNWIND_HINT_FUNC
800 movq %rdi, %rsp /* we don't return, adjust the stack frame */
801 UNWIND_HINT_REGS
802
803 call xen_pv_evtchn_do_upcall
804
805 jmp error_return
806SYM_CODE_END(exc_xen_hypervisor_callback)
807
808/*
809 * Hypervisor uses this for application faults while it executes.
810 * We get here for two reasons:
811 * 1. Fault while reloading DS, ES, FS or GS
812 * 2. Fault while executing IRET
813 * Category 1 we do not need to fix up as Xen has already reloaded all segment
814 * registers that could be reloaded and zeroed the others.
815 * Category 2 we fix up by killing the current process. We cannot use the
816 * normal Linux return path in this case because if we use the IRET hypercall
817 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
818 * We distinguish between categories by comparing each saved segment register
819 * with its current contents: any discrepancy means we in category 1.
820 */
821 __FUNC_ALIGN
822SYM_CODE_START_NOALIGN(xen_failsafe_callback)
823 UNWIND_HINT_UNDEFINED
824 ENDBR
825 movl %ds, %ecx
826 cmpw %cx, 0x10(%rsp)
827 jne 1f
828 movl %es, %ecx
829 cmpw %cx, 0x18(%rsp)
830 jne 1f
831 movl %fs, %ecx
832 cmpw %cx, 0x20(%rsp)
833 jne 1f
834 movl %gs, %ecx
835 cmpw %cx, 0x28(%rsp)
836 jne 1f
837 /* All segments match their saved values => Category 2 (Bad IRET). */
838 movq (%rsp), %rcx
839 movq 8(%rsp), %r11
840 addq $0x30, %rsp
841 pushq $0 /* RIP */
842 UNWIND_HINT_IRET_REGS offset=8
843 jmp asm_exc_general_protection
8441: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
845 movq (%rsp), %rcx
846 movq 8(%rsp), %r11
847 addq $0x30, %rsp
848 UNWIND_HINT_IRET_REGS
849 pushq $-1 /* orig_ax = -1 => not a system call */
850 PUSH_AND_CLEAR_REGS
851 ENCODE_FRAME_POINTER
852 jmp error_return
853SYM_CODE_END(xen_failsafe_callback)
854#endif /* CONFIG_XEN_PV */
855
856/*
857 * Save all registers in pt_regs. Return GSBASE related information
858 * in EBX depending on the availability of the FSGSBASE instructions:
859 *
860 * FSGSBASE R/EBX
861 * N 0 -> SWAPGS on exit
862 * 1 -> no SWAPGS on exit
863 *
864 * Y GSBASE value at entry, must be restored in paranoid_exit
865 *
866 * R14 - old CR3
867 * R15 - old SPEC_CTRL
868 */
869SYM_CODE_START(paranoid_entry)
870 ANNOTATE_NOENDBR
871 UNWIND_HINT_FUNC
872 PUSH_AND_CLEAR_REGS save_ret=1
873 ENCODE_FRAME_POINTER 8
874
875 /*
876 * Always stash CR3 in %r14. This value will be restored,
877 * verbatim, at exit. Needed if paranoid_entry interrupted
878 * another entry that already switched to the user CR3 value
879 * but has not yet returned to userspace.
880 *
881 * This is also why CS (stashed in the "iret frame" by the
882 * hardware at entry) can not be used: this may be a return
883 * to kernel code, but with a user CR3 value.
884 *
885 * Switching CR3 does not depend on kernel GSBASE so it can
886 * be done before switching to the kernel GSBASE. This is
887 * required for FSGSBASE because the kernel GSBASE has to
888 * be retrieved from a kernel internal table.
889 */
890 SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
891
892 /*
893 * Handling GSBASE depends on the availability of FSGSBASE.
894 *
895 * Without FSGSBASE the kernel enforces that negative GSBASE
896 * values indicate kernel GSBASE. With FSGSBASE no assumptions
897 * can be made about the GSBASE value when entering from user
898 * space.
899 */
900 ALTERNATIVE "jmp .Lparanoid_entry_checkgs", "", X86_FEATURE_FSGSBASE
901
902 /*
903 * Read the current GSBASE and store it in %rbx unconditionally,
904 * retrieve and set the current CPUs kernel GSBASE. The stored value
905 * has to be restored in paranoid_exit unconditionally.
906 *
907 * The unconditional write to GS base below ensures that no subsequent
908 * loads based on a mispredicted GS base can happen, therefore no LFENCE
909 * is needed here.
910 */
911 SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx
912 jmp .Lparanoid_gsbase_done
913
914.Lparanoid_entry_checkgs:
915 /* EBX = 1 -> kernel GSBASE active, no restore required */
916 movl $1, %ebx
917
918 /*
919 * The kernel-enforced convention is a negative GSBASE indicates
920 * a kernel value. No SWAPGS needed on entry and exit.
921 */
922 movl $MSR_GS_BASE, %ecx
923 rdmsr
924 testl %edx, %edx
925 js .Lparanoid_kernel_gsbase
926
927 /* EBX = 0 -> SWAPGS required on exit */
928 xorl %ebx, %ebx
929 swapgs
930.Lparanoid_kernel_gsbase:
931 FENCE_SWAPGS_KERNEL_ENTRY
932.Lparanoid_gsbase_done:
933
934 /*
935 * Once we have CR3 and %GS setup save and set SPEC_CTRL. Just like
936 * CR3 above, keep the old value in a callee saved register.
937 */
938 IBRS_ENTER save_reg=%r15
939 UNTRAIN_RET_FROM_CALL
940
941 RET
942SYM_CODE_END(paranoid_entry)
943
944/*
945 * "Paranoid" exit path from exception stack. This is invoked
946 * only on return from non-NMI IST interrupts that came
947 * from kernel space.
948 *
949 * We may be returning to very strange contexts (e.g. very early
950 * in syscall entry), so checking for preemption here would
951 * be complicated. Fortunately, there's no good reason to try
952 * to handle preemption here.
953 *
954 * R/EBX contains the GSBASE related information depending on the
955 * availability of the FSGSBASE instructions:
956 *
957 * FSGSBASE R/EBX
958 * N 0 -> SWAPGS on exit
959 * 1 -> no SWAPGS on exit
960 *
961 * Y User space GSBASE, must be restored unconditionally
962 *
963 * R14 - old CR3
964 * R15 - old SPEC_CTRL
965 */
966SYM_CODE_START_LOCAL(paranoid_exit)
967 UNWIND_HINT_REGS
968
969 /*
970 * Must restore IBRS state before both CR3 and %GS since we need access
971 * to the per-CPU x86_spec_ctrl_shadow variable.
972 */
973 IBRS_EXIT save_reg=%r15
974
975 /*
976 * The order of operations is important. PARANOID_RESTORE_CR3 requires
977 * kernel GSBASE.
978 *
979 * NB to anyone to try to optimize this code: this code does
980 * not execute at all for exceptions from user mode. Those
981 * exceptions go through error_return instead.
982 */
983 PARANOID_RESTORE_CR3 scratch_reg=%rax save_reg=%r14
984
985 /* Handle the three GSBASE cases */
986 ALTERNATIVE "jmp .Lparanoid_exit_checkgs", "", X86_FEATURE_FSGSBASE
987
988 /* With FSGSBASE enabled, unconditionally restore GSBASE */
989 wrgsbase %rbx
990 jmp restore_regs_and_return_to_kernel
991
992.Lparanoid_exit_checkgs:
993 /* On non-FSGSBASE systems, conditionally do SWAPGS */
994 testl %ebx, %ebx
995 jnz restore_regs_and_return_to_kernel
996
997 /* We are returning to a context with user GSBASE */
998 swapgs
999 jmp restore_regs_and_return_to_kernel
1000SYM_CODE_END(paranoid_exit)
1001
1002/*
1003 * Switch GS and CR3 if needed.
1004 */
1005SYM_CODE_START(error_entry)
1006 ANNOTATE_NOENDBR
1007 UNWIND_HINT_FUNC
1008
1009 PUSH_AND_CLEAR_REGS save_ret=1
1010 ENCODE_FRAME_POINTER 8
1011
1012 testb $3, CS+8(%rsp)
1013 jz .Lerror_kernelspace
1014
1015 /*
1016 * We entered from user mode or we're pretending to have entered
1017 * from user mode due to an IRET fault.
1018 */
1019 swapgs
1020 FENCE_SWAPGS_USER_ENTRY
1021 /* We have user CR3. Change to kernel CR3. */
1022 SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1023 IBRS_ENTER
1024 UNTRAIN_RET_FROM_CALL
1025
1026 leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
1027 /* Put us onto the real thread stack. */
1028 jmp sync_regs
1029
1030 /*
1031 * There are two places in the kernel that can potentially fault with
1032 * usergs. Handle them here. B stepping K8s sometimes report a
1033 * truncated RIP for IRET exceptions returning to compat mode. Check
1034 * for these here too.
1035 */
1036.Lerror_kernelspace:
1037 leaq native_irq_return_iret(%rip), %rcx
1038 cmpq %rcx, RIP+8(%rsp)
1039 je .Lerror_bad_iret
1040 movl %ecx, %eax /* zero extend */
1041 cmpq %rax, RIP+8(%rsp)
1042 je .Lbstep_iret
1043 cmpq $.Lgs_change, RIP+8(%rsp)
1044 jne .Lerror_entry_done_lfence
1045
1046 /*
1047 * hack: .Lgs_change can fail with user gsbase. If this happens, fix up
1048 * gsbase and proceed. We'll fix up the exception and land in
1049 * .Lgs_change's error handler with kernel gsbase.
1050 */
1051 swapgs
1052
1053 /*
1054 * Issue an LFENCE to prevent GS speculation, regardless of whether it is a
1055 * kernel or user gsbase.
1056 */
1057.Lerror_entry_done_lfence:
1058 FENCE_SWAPGS_KERNEL_ENTRY
1059 CALL_DEPTH_ACCOUNT
1060 leaq 8(%rsp), %rax /* return pt_regs pointer */
1061 VALIDATE_UNRET_END
1062 RET
1063
1064.Lbstep_iret:
1065 /* Fix truncated RIP */
1066 movq %rcx, RIP+8(%rsp)
1067 /* fall through */
1068
1069.Lerror_bad_iret:
1070 /*
1071 * We came from an IRET to user mode, so we have user
1072 * gsbase and CR3. Switch to kernel gsbase and CR3:
1073 */
1074 swapgs
1075 FENCE_SWAPGS_USER_ENTRY
1076 SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1077 IBRS_ENTER
1078 UNTRAIN_RET_FROM_CALL
1079
1080 /*
1081 * Pretend that the exception came from user mode: set up pt_regs
1082 * as if we faulted immediately after IRET.
1083 */
1084 leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
1085 call fixup_bad_iret
1086 mov %rax, %rdi
1087 jmp sync_regs
1088SYM_CODE_END(error_entry)
1089
1090SYM_CODE_START_LOCAL(error_return)
1091 UNWIND_HINT_REGS
1092 DEBUG_ENTRY_ASSERT_IRQS_OFF
1093 testb $3, CS(%rsp)
1094 jz restore_regs_and_return_to_kernel
1095 jmp swapgs_restore_regs_and_return_to_usermode
1096SYM_CODE_END(error_return)
1097
1098/*
1099 * Runs on exception stack. Xen PV does not go through this path at all,
1100 * so we can use real assembly here.
1101 *
1102 * Registers:
1103 * %r14: Used to save/restore the CR3 of the interrupted context
1104 * when MITIGATION_PAGE_TABLE_ISOLATION is in use. Do not clobber.
1105 */
1106SYM_CODE_START(asm_exc_nmi)
1107 UNWIND_HINT_IRET_ENTRY
1108 ENDBR
1109
1110 /*
1111 * We allow breakpoints in NMIs. If a breakpoint occurs, then
1112 * the iretq it performs will take us out of NMI context.
1113 * This means that we can have nested NMIs where the next
1114 * NMI is using the top of the stack of the previous NMI. We
1115 * can't let it execute because the nested NMI will corrupt the
1116 * stack of the previous NMI. NMI handlers are not re-entrant
1117 * anyway.
1118 *
1119 * To handle this case we do the following:
1120 * Check a special location on the stack that contains a
1121 * variable that is set when NMIs are executing.
1122 * The interrupted task's stack is also checked to see if it
1123 * is an NMI stack.
1124 * If the variable is not set and the stack is not the NMI
1125 * stack then:
1126 * o Set the special variable on the stack
1127 * o Copy the interrupt frame into an "outermost" location on the
1128 * stack
1129 * o Copy the interrupt frame into an "iret" location on the stack
1130 * o Continue processing the NMI
1131 * If the variable is set or the previous stack is the NMI stack:
1132 * o Modify the "iret" location to jump to the repeat_nmi
1133 * o return back to the first NMI
1134 *
1135 * Now on exit of the first NMI, we first clear the stack variable
1136 * The NMI stack will tell any nested NMIs at that point that it is
1137 * nested. Then we pop the stack normally with iret, and if there was
1138 * a nested NMI that updated the copy interrupt stack frame, a
1139 * jump will be made to the repeat_nmi code that will handle the second
1140 * NMI.
1141 *
1142 * However, espfix prevents us from directly returning to userspace
1143 * with a single IRET instruction. Similarly, IRET to user mode
1144 * can fault. We therefore handle NMIs from user space like
1145 * other IST entries.
1146 */
1147
1148 ASM_CLAC
1149 cld
1150
1151 /* Use %rdx as our temp variable throughout */
1152 pushq %rdx
1153
1154 testb $3, CS-RIP+8(%rsp)
1155 jz .Lnmi_from_kernel
1156
1157 /*
1158 * NMI from user mode. We need to run on the thread stack, but we
1159 * can't go through the normal entry paths: NMIs are masked, and
1160 * we don't want to enable interrupts, because then we'll end
1161 * up in an awkward situation in which IRQs are on but NMIs
1162 * are off.
1163 *
1164 * We also must not push anything to the stack before switching
1165 * stacks lest we corrupt the "NMI executing" variable.
1166 */
1167
1168 swapgs
1169 FENCE_SWAPGS_USER_ENTRY
1170 SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
1171 movq %rsp, %rdx
1172 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
1173 UNWIND_HINT_IRET_REGS base=%rdx offset=8
1174 pushq 5*8(%rdx) /* pt_regs->ss */
1175 pushq 4*8(%rdx) /* pt_regs->rsp */
1176 pushq 3*8(%rdx) /* pt_regs->flags */
1177 pushq 2*8(%rdx) /* pt_regs->cs */
1178 pushq 1*8(%rdx) /* pt_regs->rip */
1179 UNWIND_HINT_IRET_REGS
1180 pushq $-1 /* pt_regs->orig_ax */
1181 PUSH_AND_CLEAR_REGS rdx=(%rdx)
1182 ENCODE_FRAME_POINTER
1183
1184 IBRS_ENTER
1185 UNTRAIN_RET
1186
1187 /*
1188 * At this point we no longer need to worry about stack damage
1189 * due to nesting -- we're on the normal thread stack and we're
1190 * done with the NMI stack.
1191 */
1192
1193 movq %rsp, %rdi
1194 call exc_nmi
1195
1196 /*
1197 * Return back to user mode. We must *not* do the normal exit
1198 * work, because we don't want to enable interrupts.
1199 */
1200 jmp swapgs_restore_regs_and_return_to_usermode
1201
1202.Lnmi_from_kernel:
1203 /*
1204 * Here's what our stack frame will look like:
1205 * +---------------------------------------------------------+
1206 * | original SS |
1207 * | original Return RSP |
1208 * | original RFLAGS |
1209 * | original CS |
1210 * | original RIP |
1211 * +---------------------------------------------------------+
1212 * | temp storage for rdx |
1213 * +---------------------------------------------------------+
1214 * | "NMI executing" variable |
1215 * +---------------------------------------------------------+
1216 * | iret SS } Copied from "outermost" frame |
1217 * | iret Return RSP } on each loop iteration; overwritten |
1218 * | iret RFLAGS } by a nested NMI to force another |
1219 * | iret CS } iteration if needed. |
1220 * | iret RIP } |
1221 * +---------------------------------------------------------+
1222 * | outermost SS } initialized in first_nmi; |
1223 * | outermost Return RSP } will not be changed before |
1224 * | outermost RFLAGS } NMI processing is done. |
1225 * | outermost CS } Copied to "iret" frame on each |
1226 * | outermost RIP } iteration. |
1227 * +---------------------------------------------------------+
1228 * | pt_regs |
1229 * +---------------------------------------------------------+
1230 *
1231 * The "original" frame is used by hardware. Before re-enabling
1232 * NMIs, we need to be done with it, and we need to leave enough
1233 * space for the asm code here.
1234 *
1235 * We return by executing IRET while RSP points to the "iret" frame.
1236 * That will either return for real or it will loop back into NMI
1237 * processing.
1238 *
1239 * The "outermost" frame is copied to the "iret" frame on each
1240 * iteration of the loop, so each iteration starts with the "iret"
1241 * frame pointing to the final return target.
1242 */
1243
1244 /*
1245 * Determine whether we're a nested NMI.
1246 *
1247 * If we interrupted kernel code between repeat_nmi and
1248 * end_repeat_nmi, then we are a nested NMI. We must not
1249 * modify the "iret" frame because it's being written by
1250 * the outer NMI. That's okay; the outer NMI handler is
1251 * about to call exc_nmi() anyway, so we can just resume
1252 * the outer NMI.
1253 */
1254
1255 movq $repeat_nmi, %rdx
1256 cmpq 8(%rsp), %rdx
1257 ja 1f
1258 movq $end_repeat_nmi, %rdx
1259 cmpq 8(%rsp), %rdx
1260 ja nested_nmi_out
12611:
1262
1263 /*
1264 * Now check "NMI executing". If it's set, then we're nested.
1265 * This will not detect if we interrupted an outer NMI just
1266 * before IRET.
1267 */
1268 cmpl $1, -8(%rsp)
1269 je nested_nmi
1270
1271 /*
1272 * Now test if the previous stack was an NMI stack. This covers
1273 * the case where we interrupt an outer NMI after it clears
1274 * "NMI executing" but before IRET. We need to be careful, though:
1275 * there is one case in which RSP could point to the NMI stack
1276 * despite there being no NMI active: naughty userspace controls
1277 * RSP at the very beginning of the SYSCALL targets. We can
1278 * pull a fast one on naughty userspace, though: we program
1279 * SYSCALL to mask DF, so userspace cannot cause DF to be set
1280 * if it controls the kernel's RSP. We set DF before we clear
1281 * "NMI executing".
1282 */
1283 lea 6*8(%rsp), %rdx
1284 /* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
1285 cmpq %rdx, 4*8(%rsp)
1286 /* If the stack pointer is above the NMI stack, this is a normal NMI */
1287 ja first_nmi
1288
1289 subq $EXCEPTION_STKSZ, %rdx
1290 cmpq %rdx, 4*8(%rsp)
1291 /* If it is below the NMI stack, it is a normal NMI */
1292 jb first_nmi
1293
1294 /* Ah, it is within the NMI stack. */
1295
1296 testb $(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
1297 jz first_nmi /* RSP was user controlled. */
1298
1299 /* This is a nested NMI. */
1300
1301nested_nmi:
1302 /*
1303 * Modify the "iret" frame to point to repeat_nmi, forcing another
1304 * iteration of NMI handling.
1305 */
1306 subq $8, %rsp
1307 leaq -10*8(%rsp), %rdx
1308 pushq $__KERNEL_DS
1309 pushq %rdx
1310 pushfq
1311 pushq $__KERNEL_CS
1312 pushq $repeat_nmi
1313
1314 /* Put stack back */
1315 addq $(6*8), %rsp
1316
1317nested_nmi_out:
1318 popq %rdx
1319
1320 /* We are returning to kernel mode, so this cannot result in a fault. */
1321 iretq
1322
1323first_nmi:
1324 /* Restore rdx. */
1325 movq (%rsp), %rdx
1326
1327 /* Make room for "NMI executing". */
1328 pushq $0
1329
1330 /* Leave room for the "iret" frame */
1331 subq $(5*8), %rsp
1332
1333 /* Copy the "original" frame to the "outermost" frame */
1334 .rept 5
1335 pushq 11*8(%rsp)
1336 .endr
1337 UNWIND_HINT_IRET_REGS
1338
1339 /* Everything up to here is safe from nested NMIs */
1340
1341#ifdef CONFIG_DEBUG_ENTRY
1342 /*
1343 * For ease of testing, unmask NMIs right away. Disabled by
1344 * default because IRET is very expensive.
1345 */
1346 pushq $0 /* SS */
1347 pushq %rsp /* RSP (minus 8 because of the previous push) */
1348 addq $8, (%rsp) /* Fix up RSP */
1349 pushfq /* RFLAGS */
1350 pushq $__KERNEL_CS /* CS */
1351 pushq $1f /* RIP */
1352 iretq /* continues at repeat_nmi below */
1353 UNWIND_HINT_IRET_REGS
13541:
1355#endif
1356
1357repeat_nmi:
1358 ANNOTATE_NOENDBR // this code
1359 /*
1360 * If there was a nested NMI, the first NMI's iret will return
1361 * here. But NMIs are still enabled and we can take another
1362 * nested NMI. The nested NMI checks the interrupted RIP to see
1363 * if it is between repeat_nmi and end_repeat_nmi, and if so
1364 * it will just return, as we are about to repeat an NMI anyway.
1365 * This makes it safe to copy to the stack frame that a nested
1366 * NMI will update.
1367 *
1368 * RSP is pointing to "outermost RIP". gsbase is unknown, but, if
1369 * we're repeating an NMI, gsbase has the same value that it had on
1370 * the first iteration. paranoid_entry will load the kernel
1371 * gsbase if needed before we call exc_nmi(). "NMI executing"
1372 * is zero.
1373 */
1374 movq $1, 10*8(%rsp) /* Set "NMI executing". */
1375
1376 /*
1377 * Copy the "outermost" frame to the "iret" frame. NMIs that nest
1378 * here must not modify the "iret" frame while we're writing to
1379 * it or it will end up containing garbage.
1380 */
1381 addq $(10*8), %rsp
1382 .rept 5
1383 pushq -6*8(%rsp)
1384 .endr
1385 subq $(5*8), %rsp
1386end_repeat_nmi:
1387 ANNOTATE_NOENDBR // this code
1388
1389 /*
1390 * Everything below this point can be preempted by a nested NMI.
1391 * If this happens, then the inner NMI will change the "iret"
1392 * frame to point back to repeat_nmi.
1393 */
1394 pushq $-1 /* ORIG_RAX: no syscall to restart */
1395
1396 /*
1397 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1398 * as we should not be calling schedule in NMI context.
1399 * Even with normal interrupts enabled. An NMI should not be
1400 * setting NEED_RESCHED or anything that normal interrupts and
1401 * exceptions might do.
1402 */
1403 call paranoid_entry
1404 UNWIND_HINT_REGS
1405
1406 movq %rsp, %rdi
1407 call exc_nmi
1408
1409 /* Always restore stashed SPEC_CTRL value (see paranoid_entry) */
1410 IBRS_EXIT save_reg=%r15
1411
1412 PARANOID_RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
1413
1414 /*
1415 * The above invocation of paranoid_entry stored the GSBASE
1416 * related information in R/EBX depending on the availability
1417 * of FSGSBASE.
1418 *
1419 * If FSGSBASE is enabled, restore the saved GSBASE value
1420 * unconditionally, otherwise take the conditional SWAPGS path.
1421 */
1422 ALTERNATIVE "jmp nmi_no_fsgsbase", "", X86_FEATURE_FSGSBASE
1423
1424 wrgsbase %rbx
1425 jmp nmi_restore
1426
1427nmi_no_fsgsbase:
1428 /* EBX == 0 -> invoke SWAPGS */
1429 testl %ebx, %ebx
1430 jnz nmi_restore
1431
1432nmi_swapgs:
1433 swapgs
1434
1435nmi_restore:
1436 POP_REGS
1437
1438 /*
1439 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
1440 * at the "iret" frame.
1441 */
1442 addq $6*8, %rsp
1443
1444 /*
1445 * Clear "NMI executing". Set DF first so that we can easily
1446 * distinguish the remaining code between here and IRET from
1447 * the SYSCALL entry and exit paths.
1448 *
1449 * We arguably should just inspect RIP instead, but I (Andy) wrote
1450 * this code when I had the misapprehension that Xen PV supported
1451 * NMIs, and Xen PV would break that approach.
1452 */
1453 std
1454 movq $0, 5*8(%rsp) /* clear "NMI executing" */
1455
1456 /*
1457 * Skip CLEAR_CPU_BUFFERS here, since it only helps in rare cases like
1458 * NMI in kernel after user state is restored. For an unprivileged user
1459 * these conditions are hard to meet.
1460 */
1461
1462 /*
1463 * iretq reads the "iret" frame and exits the NMI stack in a
1464 * single instruction. We are returning to kernel mode, so this
1465 * cannot result in a fault. Similarly, we don't need to worry
1466 * about espfix64 on the way back to kernel mode.
1467 */
1468 iretq
1469SYM_CODE_END(asm_exc_nmi)
1470
1471/*
1472 * This handles SYSCALL from 32-bit code. There is no way to program
1473 * MSRs to fully disable 32-bit SYSCALL.
1474 */
1475SYM_CODE_START(entry_SYSCALL32_ignore)
1476 UNWIND_HINT_END_OF_STACK
1477 ENDBR
1478 mov $-ENOSYS, %eax
1479 CLEAR_CPU_BUFFERS
1480 sysretl
1481SYM_CODE_END(entry_SYSCALL32_ignore)
1482
1483.pushsection .text, "ax"
1484 __FUNC_ALIGN
1485SYM_CODE_START_NOALIGN(rewind_stack_and_make_dead)
1486 UNWIND_HINT_FUNC
1487 /* Prevent any naive code from trying to unwind to our caller. */
1488 xorl %ebp, %ebp
1489
1490 movq PER_CPU_VAR(cpu_current_top_of_stack), %rax
1491 leaq -PTREGS_SIZE(%rax), %rsp
1492 UNWIND_HINT_REGS
1493
1494 call make_task_dead
1495SYM_CODE_END(rewind_stack_and_make_dead)
1496.popsection
1497
1498/*
1499 * This sequence executes branches in order to remove user branch information
1500 * from the branch history tracker in the Branch Predictor, therefore removing
1501 * user influence on subsequent BTB lookups.
1502 *
1503 * It should be used on parts prior to Alder Lake. Newer parts should use the
1504 * BHI_DIS_S hardware control instead. If a pre-Alder Lake part is being
1505 * virtualized on newer hardware the VMM should protect against BHI attacks by
1506 * setting BHI_DIS_S for the guests.
1507 *
1508 * CALLs/RETs are necessary to prevent Loop Stream Detector(LSD) from engaging
1509 * and not clearing the branch history. The call tree looks like:
1510 *
1511 * call 1
1512 * call 2
1513 * call 2
1514 * call 2
1515 * call 2
1516 * call 2
1517 * ret
1518 * ret
1519 * ret
1520 * ret
1521 * ret
1522 * ret
1523 *
1524 * This means that the stack is non-constant and ORC can't unwind it with %rsp
1525 * alone. Therefore we unconditionally set up the frame pointer, which allows
1526 * ORC to unwind properly.
1527 *
1528 * The alignment is for performance and not for safety, and may be safely
1529 * refactored in the future if needed. The .skips are for safety, to ensure
1530 * that all RETs are in the second half of a cacheline to mitigate Indirect
1531 * Target Selection, rather than taking the slowpath via its_return_thunk.
1532 */
1533SYM_FUNC_START(clear_bhb_loop)
1534 ANNOTATE_NOENDBR
1535 push %rbp
1536 mov %rsp, %rbp
1537 movl $5, %ecx
1538 ANNOTATE_INTRA_FUNCTION_CALL
1539 call 1f
1540 jmp 5f
1541 .align 64, 0xcc
1542 /*
1543 * Shift instructions so that the RET is in the upper half of the
1544 * cacheline and don't take the slowpath to its_return_thunk.
1545 */
1546 .skip 32 - (.Lret1 - 1f), 0xcc
1547 ANNOTATE_INTRA_FUNCTION_CALL
15481: call 2f
1549.Lret1: RET
1550 .align 64, 0xcc
1551 /*
1552 * As above shift instructions for RET at .Lret2 as well.
1553 *
1554 * This should be ideally be: .skip 32 - (.Lret2 - 2f), 0xcc
1555 * but some Clang versions (e.g. 18) don't like this.
1556 */
1557 .skip 32 - 18, 0xcc
15582: movl $5, %eax
15593: jmp 4f
1560 nop
15614: sub $1, %eax
1562 jnz 3b
1563 sub $1, %ecx
1564 jnz 1b
1565.Lret2: RET
15665: lfence
1567 pop %rbp
1568 RET
1569SYM_FUNC_END(clear_bhb_loop)
1570EXPORT_SYMBOL_FOR_KVM(clear_bhb_loop)
1571STACK_FRAME_NON_STANDARD(clear_bhb_loop)
1572

source code of linux/arch/x86/entry/entry_64.S