1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | |
3 | #include <linux/compat.h> |
4 | #include <linux/context_tracking.h> |
5 | #include <linux/randomize_kstack.h> |
6 | |
7 | #include <asm/interrupt.h> |
8 | #include <asm/kup.h> |
9 | #include <asm/syscall.h> |
10 | #include <asm/time.h> |
11 | #include <asm/tm.h> |
12 | #include <asm/unistd.h> |
13 | |
14 | |
15 | /* Has to run notrace because it is entered not completely "reconciled" */ |
16 | notrace long system_call_exception(struct pt_regs *regs, unsigned long r0) |
17 | { |
18 | long ret; |
19 | syscall_fn f; |
20 | |
21 | kuap_lock(); |
22 | |
23 | add_random_kstack_offset(); |
24 | |
25 | if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) |
26 | BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED); |
27 | |
28 | trace_hardirqs_off(); /* finish reconciling */ |
29 | |
30 | CT_WARN_ON(ct_state() == CONTEXT_KERNEL); |
31 | user_exit_irqoff(); |
32 | |
33 | BUG_ON(regs_is_unrecoverable(regs)); |
34 | BUG_ON(!user_mode(regs)); |
35 | BUG_ON(arch_irq_disabled_regs(regs)); |
36 | |
37 | #ifdef CONFIG_PPC_PKEY |
38 | if (mmu_has_feature(MMU_FTR_PKEY)) { |
39 | unsigned long amr, iamr; |
40 | bool flush_needed = false; |
41 | /* |
42 | * When entering from userspace we mostly have the AMR/IAMR |
43 | * different from kernel default values. Hence don't compare. |
44 | */ |
45 | amr = mfspr(SPRN_AMR); |
46 | iamr = mfspr(SPRN_IAMR); |
47 | regs->amr = amr; |
48 | regs->iamr = iamr; |
49 | if (mmu_has_feature(MMU_FTR_KUAP)) { |
50 | mtspr(SPRN_AMR, AMR_KUAP_BLOCKED); |
51 | flush_needed = true; |
52 | } |
53 | if (mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) { |
54 | mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED); |
55 | flush_needed = true; |
56 | } |
57 | if (flush_needed) |
58 | isync(); |
59 | } else |
60 | #endif |
61 | kuap_assert_locked(); |
62 | |
63 | booke_restore_dbcr0(); |
64 | |
65 | account_cpu_user_entry(); |
66 | |
67 | account_stolen_time(); |
68 | |
69 | /* |
70 | * This is not required for the syscall exit path, but makes the |
71 | * stack frame look nicer. If this was initialised in the first stack |
72 | * frame, or if the unwinder was taught the first stack frame always |
73 | * returns to user with IRQS_ENABLED, this store could be avoided! |
74 | */ |
75 | irq_soft_mask_regs_set_state(regs, IRQS_ENABLED); |
76 | |
77 | /* |
78 | * If system call is called with TM active, set _TIF_RESTOREALL to |
79 | * prevent RFSCV being used to return to userspace, because POWER9 |
80 | * TM implementation has problems with this instruction returning to |
81 | * transactional state. Final register values are not relevant because |
82 | * the transaction will be aborted upon return anyway. Or in the case |
83 | * of unsupported_scv SIGILL fault, the return state does not much |
84 | * matter because it's an edge case. |
85 | */ |
86 | if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && |
87 | unlikely(MSR_TM_TRANSACTIONAL(regs->msr))) |
88 | set_bits(_TIF_RESTOREALL, ¤t_thread_info()->flags); |
89 | |
90 | /* |
91 | * If the system call was made with a transaction active, doom it and |
92 | * return without performing the system call. Unless it was an |
93 | * unsupported scv vector, in which case it's treated like an illegal |
94 | * instruction. |
95 | */ |
96 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
97 | if (unlikely(MSR_TM_TRANSACTIONAL(regs->msr)) && |
98 | !trap_is_unsupported_scv(regs)) { |
99 | /* Enable TM in the kernel, and disable EE (for scv) */ |
100 | hard_irq_disable(); |
101 | mtmsr(mfmsr() | MSR_TM); |
102 | |
103 | /* tabort, this dooms the transaction, nothing else */ |
104 | asm volatile(".long 0x7c00071d | ((%0) << 16)" |
105 | :: "r" (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)); |
106 | |
107 | /* |
108 | * Userspace will never see the return value. Execution will |
109 | * resume after the tbegin. of the aborted transaction with the |
110 | * checkpointed register state. A context switch could occur |
111 | * or signal delivered to the process before resuming the |
112 | * doomed transaction context, but that should all be handled |
113 | * as expected. |
114 | */ |
115 | return -ENOSYS; |
116 | } |
117 | #endif // CONFIG_PPC_TRANSACTIONAL_MEM |
118 | |
119 | local_irq_enable(); |
120 | |
121 | if (unlikely(read_thread_flags() & _TIF_SYSCALL_DOTRACE)) { |
122 | if (unlikely(trap_is_unsupported_scv(regs))) { |
123 | /* Unsupported scv vector */ |
124 | _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); |
125 | return regs->gpr[3]; |
126 | } |
127 | /* |
128 | * We use the return value of do_syscall_trace_enter() as the |
129 | * syscall number. If the syscall was rejected for any reason |
130 | * do_syscall_trace_enter() returns an invalid syscall number |
131 | * and the test against NR_syscalls will fail and the return |
132 | * value to be used is in regs->gpr[3]. |
133 | */ |
134 | r0 = do_syscall_trace_enter(regs); |
135 | if (unlikely(r0 >= NR_syscalls)) |
136 | return regs->gpr[3]; |
137 | |
138 | } else if (unlikely(r0 >= NR_syscalls)) { |
139 | if (unlikely(trap_is_unsupported_scv(regs))) { |
140 | /* Unsupported scv vector */ |
141 | _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); |
142 | return regs->gpr[3]; |
143 | } |
144 | return -ENOSYS; |
145 | } |
146 | |
147 | /* May be faster to do array_index_nospec? */ |
148 | barrier_nospec(); |
149 | |
150 | #ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER |
151 | // No COMPAT if we have SYSCALL_WRAPPER, see Kconfig |
152 | f = (void *)sys_call_table[r0]; |
153 | ret = f(regs); |
154 | #else |
155 | if (unlikely(is_compat_task())) { |
156 | unsigned long r3, r4, r5, r6, r7, r8; |
157 | |
158 | f = (void *)compat_sys_call_table[r0]; |
159 | |
160 | r3 = regs->gpr[3] & 0x00000000ffffffffULL; |
161 | r4 = regs->gpr[4] & 0x00000000ffffffffULL; |
162 | r5 = regs->gpr[5] & 0x00000000ffffffffULL; |
163 | r6 = regs->gpr[6] & 0x00000000ffffffffULL; |
164 | r7 = regs->gpr[7] & 0x00000000ffffffffULL; |
165 | r8 = regs->gpr[8] & 0x00000000ffffffffULL; |
166 | |
167 | ret = f(r3, r4, r5, r6, r7, r8); |
168 | } else { |
169 | f = (void *)sys_call_table[r0]; |
170 | |
171 | ret = f(regs->gpr[3], regs->gpr[4], regs->gpr[5], |
172 | regs->gpr[6], regs->gpr[7], regs->gpr[8]); |
173 | } |
174 | #endif |
175 | |
176 | /* |
177 | * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(), |
178 | * so the maximum stack offset is 1k bytes (10 bits). |
179 | * |
180 | * The actual entropy will be further reduced by the compiler when |
181 | * applying stack alignment constraints: the powerpc architecture |
182 | * may have two kinds of stack alignment (16-bytes and 8-bytes). |
183 | * |
184 | * So the resulting 6 or 7 bits of entropy is seen in SP[9:4] or SP[9:3]. |
185 | */ |
186 | choose_random_kstack_offset(mftb()); |
187 | |
188 | return ret; |
189 | } |
190 | |