1/* SPDX-License-Identifier: GPL-2.0-only */
2#include <linux/export.h>
3#include <asm/asm-offsets.h>
4#include <asm/cache.h>
5#include <asm/code-patching-asm.h>
6#include <asm/exception-64s.h>
7#include <asm/kvm_asm.h>
8#include <asm/kvm_book3s_asm.h>
9#include <asm/mmu.h>
10#include <asm/ppc_asm.h>
11#include <asm/ptrace.h>
12#include <asm/reg.h>
13#include <asm/ultravisor-api.h>
14
15/*
16 * These are branched to from interrupt handlers in exception-64s.S which set
17 * IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero.
18 */
19
20/*
21 * This is a hcall, so register convention is as
22 * Documentation/arch/powerpc/papr_hcalls.rst.
23 *
24 * This may also be a syscall from PR-KVM userspace that is to be
25 * reflected to the PR guest kernel, so registers may be set up for
26 * a system call rather than hcall. We don't currently clobber
27 * anything here, but the 0xc00 handler has already clobbered CTR
28 * and CR0, so PR-KVM can not support a guest kernel that preserves
29 * those registers across its system calls.
30 *
31 * The state of registers is as kvmppc_interrupt, except CFAR is not
32 * saved, R13 is not in SCRATCH0, and R10 does not contain the trap.
33 */
34.global kvmppc_hcall
35.balign IFETCH_ALIGN_BYTES
36kvmppc_hcall:
37#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
38 lbz r10,HSTATE_IN_GUEST(r13)
39 cmpwi r10,KVM_GUEST_MODE_HV_P9
40 beq kvmppc_p9_exit_hcall
41#endif
42 ld r10,PACA_EXGEN+EX_R13(r13)
43 SET_SCRATCH0(r10)
44 li r10,0xc00
45 /* Now we look like kvmppc_interrupt */
46 li r11,PACA_EXGEN
47 b .Lgot_save_area
48
49/*
50 * KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that
51 * call convention:
52 *
53 * guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area
54 * guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts
55 * guest R13 also saved in SCRATCH0
56 * R13 = PACA
57 * R11 = (H)SRR0
58 * R12 = (H)SRR1
59 * R9 = guest CR
60 * PPR is set to medium
61 *
62 * With the addition for KVM:
63 * R10 = trap vector
64 */
65.global kvmppc_interrupt
66.balign IFETCH_ALIGN_BYTES
67kvmppc_interrupt:
68#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
69 std r10,HSTATE_SCRATCH0(r13)
70 lbz r10,HSTATE_IN_GUEST(r13)
71 cmpwi r10,KVM_GUEST_MODE_HV_P9
72 beq kvmppc_p9_exit_interrupt
73 ld r10,HSTATE_SCRATCH0(r13)
74#endif
75 li r11,PACA_EXGEN
76 cmpdi r10,0x200
77 bgt+ .Lgot_save_area
78 li r11,PACA_EXMC
79 beq .Lgot_save_area
80 li r11,PACA_EXNMI
81.Lgot_save_area:
82 add r11,r11,r13
83BEGIN_FTR_SECTION
84 ld r12,EX_CFAR(r11)
85 std r12,HSTATE_CFAR(r13)
86END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
87 ld r12,EX_CTR(r11)
88 mtctr r12
89BEGIN_FTR_SECTION
90 ld r12,EX_PPR(r11)
91 std r12,HSTATE_PPR(r13)
92END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
93 ld r12,EX_R12(r11)
94 std r12,HSTATE_SCRATCH0(r13)
95 sldi r12,r9,32
96 or r12,r12,r10
97 ld r9,EX_R9(r11)
98 ld r10,EX_R10(r11)
99 ld r11,EX_R11(r11)
100
101 /*
102 * Hcalls and other interrupts come here after normalising register
103 * contents and save locations:
104 *
105 * R12 = (guest CR << 32) | interrupt vector
106 * R13 = PACA
107 * guest R12 saved in shadow HSTATE_SCRATCH0
108 * guest R13 saved in SPRN_SCRATCH0
109 */
110 std r9,HSTATE_SCRATCH2(r13)
111 lbz r9,HSTATE_IN_GUEST(r13)
112 cmpwi r9,KVM_GUEST_MODE_SKIP
113 beq- .Lmaybe_skip
114.Lno_skip:
115#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
116#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
117 cmpwi r9,KVM_GUEST_MODE_GUEST
118 beq kvmppc_interrupt_pr
119#endif
120 b kvmppc_interrupt_hv
121#else
122 b kvmppc_interrupt_pr
123#endif
124
125/*
126 * "Skip" interrupts are part of a trick KVM uses a with hash guests to load
127 * the faulting instruction in guest memory from the hypervisor without
128 * walking page tables.
129 *
130 * When the guest takes a fault that requires the hypervisor to load the
131 * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1
132 * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets
133 * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions
134 * but loads and stores will access the guest context. This is used to load
135 * the faulting instruction using the faulting guest effective address.
136 *
137 * However the guest context may not be able to translate, or it may cause a
138 * machine check or other issue, which results in a fault in the host
139 * (even with KVM-HV).
140 *
141 * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they
142 * are (or are likely) caused by that load, the instruction is skipped by
143 * just returning with the PC advanced +4, where it is noticed the load did
144 * not execute and it goes to the slow path which walks the page tables to
145 * read guest memory.
146 */
147.Lmaybe_skip:
148 cmpwi r12,BOOK3S_INTERRUPT_MACHINE_CHECK
149 beq 1f
150 cmpwi r12,BOOK3S_INTERRUPT_DATA_STORAGE
151 beq 1f
152 cmpwi r12,BOOK3S_INTERRUPT_DATA_SEGMENT
153 beq 1f
154#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
155 /* HSRR interrupts get 2 added to interrupt number */
156 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2
157 beq 2f
158#endif
159 b .Lno_skip
1601: mfspr r9,SPRN_SRR0
161 addi r9,r9,4
162 mtspr SPRN_SRR0,r9
163 ld r12,HSTATE_SCRATCH0(r13)
164 ld r9,HSTATE_SCRATCH2(r13)
165 GET_SCRATCH0(r13)
166 RFI_TO_KERNEL
167#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1682: mfspr r9,SPRN_HSRR0
169 addi r9,r9,4
170 mtspr SPRN_HSRR0,r9
171 ld r12,HSTATE_SCRATCH0(r13)
172 ld r9,HSTATE_SCRATCH2(r13)
173 GET_SCRATCH0(r13)
174 HRFI_TO_KERNEL
175#endif
176
177#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
178
179/* Stack frame offsets for kvmppc_p9_enter_guest */
180#define SFS (144 + STACK_FRAME_MIN_SIZE)
181#define STACK_SLOT_NVGPRS (SFS - 144) /* 18 gprs */
182
183/*
184 * void kvmppc_p9_enter_guest(struct vcpu *vcpu);
185 *
186 * Enter the guest on a ISAv3.0 or later system.
187 */
188.balign IFETCH_ALIGN_BYTES
189_GLOBAL(kvmppc_p9_enter_guest)
190EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest)
191 mflr r0
192 std r0,PPC_LR_STKOFF(r1)
193 stdu r1,-SFS(r1)
194
195 std r1,HSTATE_HOST_R1(r13)
196
197 mfcr r4
198 stw r4,SFS+8(r1)
199
200 reg = 14
201 .rept 18
202 std reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
203 reg = reg + 1
204 .endr
205
206 ld r4,VCPU_LR(r3)
207 mtlr r4
208 ld r4,VCPU_CTR(r3)
209 mtctr r4
210 ld r4,VCPU_XER(r3)
211 mtspr SPRN_XER,r4
212
213 ld r1,VCPU_CR(r3)
214
215BEGIN_FTR_SECTION
216 ld r4,VCPU_CFAR(r3)
217 mtspr SPRN_CFAR,r4
218END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
219BEGIN_FTR_SECTION
220 ld r4,VCPU_PPR(r3)
221 mtspr SPRN_PPR,r4
222END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
223
224 reg = 4
225 .rept 28
226 ld reg,__VCPU_GPR(reg)(r3)
227 reg = reg + 1
228 .endr
229
230 ld r4,VCPU_KVM(r3)
231 lbz r4,KVM_SECURE_GUEST(r4)
232 cmpdi r4,0
233 ld r4,VCPU_GPR(R4)(r3)
234 bne .Lret_to_ultra
235
236 mtcr r1
237
238 ld r0,VCPU_GPR(R0)(r3)
239 ld r1,VCPU_GPR(R1)(r3)
240 ld r2,VCPU_GPR(R2)(r3)
241 ld r3,VCPU_GPR(R3)(r3)
242
243 HRFI_TO_GUEST
244 b .
245
246 /*
247 * Use UV_RETURN ultracall to return control back to the Ultravisor
248 * after processing an hypercall or interrupt that was forwarded
249 * (a.k.a. reflected) to the Hypervisor.
250 *
251 * All registers have already been reloaded except the ucall requires:
252 * R0 = hcall result
253 * R2 = SRR1, so UV can detect a synthesized interrupt (if any)
254 * R3 = UV_RETURN
255 */
256.Lret_to_ultra:
257 mtcr r1
258 ld r1,VCPU_GPR(R1)(r3)
259
260 ld r0,VCPU_GPR(R3)(r3)
261 mfspr r2,SPRN_SRR1
262 LOAD_REG_IMMEDIATE(r3, UV_RETURN)
263 sc 2
264
265/*
266 * kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from
267 * above if the interrupt was taken for a guest that was entered via
268 * kvmppc_p9_enter_guest().
269 *
270 * The exit code recovers the host stack and vcpu pointer, saves all guest GPRs
271 * and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re-
272 * establishes the host stack and registers to return from the
273 * kvmppc_p9_enter_guest() function, which saves CTR and other guest registers
274 * (SPRs and FP, VEC, etc).
275 */
276.balign IFETCH_ALIGN_BYTES
277kvmppc_p9_exit_hcall:
278 mfspr r11,SPRN_SRR0
279 mfspr r12,SPRN_SRR1
280 li r10,0xc00
281 std r10,HSTATE_SCRATCH0(r13)
282
283.balign IFETCH_ALIGN_BYTES
284kvmppc_p9_exit_interrupt:
285 /*
286 * If set to KVM_GUEST_MODE_HV_P9 but we're still in the
287 * hypervisor, that means we can't return from the entry stack.
288 */
289 rldicl. r10,r12,64-MSR_HV_LG,63
290 bne- kvmppc_p9_bad_interrupt
291
292 std r1,HSTATE_SCRATCH1(r13)
293 std r3,HSTATE_SCRATCH2(r13)
294 ld r1,HSTATE_HOST_R1(r13)
295 ld r3,HSTATE_KVM_VCPU(r13)
296
297 std r9,VCPU_CR(r3)
298
2991:
300 std r11,VCPU_PC(r3)
301 std r12,VCPU_MSR(r3)
302
303 reg = 14
304 .rept 18
305 std reg,__VCPU_GPR(reg)(r3)
306 reg = reg + 1
307 .endr
308
309 /* r1, r3, r9-r13 are saved to vcpu by C code */
310 std r0,VCPU_GPR(R0)(r3)
311 std r2,VCPU_GPR(R2)(r3)
312 reg = 4
313 .rept 5
314 std reg,__VCPU_GPR(reg)(r3)
315 reg = reg + 1
316 .endr
317
318 LOAD_PACA_TOC()
319
320 mflr r4
321 std r4,VCPU_LR(r3)
322 mfspr r4,SPRN_XER
323 std r4,VCPU_XER(r3)
324
325 reg = 14
326 .rept 18
327 ld reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
328 reg = reg + 1
329 .endr
330
331 lwz r4,SFS+8(r1)
332 mtcr r4
333
334 /*
335 * Flush the link stack here, before executing the first blr on the
336 * way out of the guest.
337 *
338 * The link stack won't match coming out of the guest anyway so the
339 * only cost is the flush itself. The call clobbers r0.
340 */
3411: nop
342 patch_site 1b patch__call_kvm_flush_link_stack_p9
343
344 addi r1,r1,SFS
345 ld r0,PPC_LR_STKOFF(r1)
346 mtlr r0
347 blr
348
349/*
350 * Took an interrupt somewhere right before HRFID to guest, so registers are
351 * in a bad way. Return things hopefully enough to run host virtual code and
352 * run the Linux interrupt handler (SRESET or MCE) to print something useful.
353 *
354 * We could be really clever and save all host registers in known locations
355 * before setting HSTATE_IN_GUEST, then restoring them all here, and setting
356 * return address to a fixup that sets them up again. But that's a lot of
357 * effort for a small bit of code. Lots of other things to do first.
358 */
359kvmppc_p9_bad_interrupt:
360BEGIN_MMU_FTR_SECTION
361 /*
362 * Hash host doesn't try to recover MMU (requires host SLB reload)
363 */
364 b .
365END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
366 /*
367 * Clean up guest registers to give host a chance to run.
368 */
369 li r10,0
370 mtspr SPRN_AMR,r10
371 mtspr SPRN_IAMR,r10
372 mtspr SPRN_CIABR,r10
373 mtspr SPRN_DAWRX0,r10
374BEGIN_FTR_SECTION
375 mtspr SPRN_DAWRX1,r10
376END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
377
378 /*
379 * Switch to host MMU mode (don't have the real host PID but we aren't
380 * going back to userspace).
381 */
382 hwsync
383 isync
384
385 mtspr SPRN_PID,r10
386
387 ld r10, HSTATE_KVM_VCPU(r13)
388 ld r10, VCPU_KVM(r10)
389 lwz r10, KVM_HOST_LPID(r10)
390 mtspr SPRN_LPID,r10
391
392 ld r10, HSTATE_KVM_VCPU(r13)
393 ld r10, VCPU_KVM(r10)
394 ld r10, KVM_HOST_LPCR(r10)
395 mtspr SPRN_LPCR,r10
396
397 isync
398
399 /*
400 * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear
401 * MSR_RI in r12 ([H]SRR1) so the handler won't try to return.
402 */
403 li r10,KVM_GUEST_MODE_NONE
404 stb r10,HSTATE_IN_GUEST(r13)
405 li r10,MSR_RI
406 andc r12,r12,r10
407
408 /*
409 * Go back to interrupt handler. MCE and SRESET have their specific
410 * PACA save area so they should be used directly. They set up their
411 * own stack. The other handlers all use EXGEN. They will use the
412 * guest r1 if it looks like a kernel stack, so just load the
413 * emergency stack and go to program check for all other interrupts.
414 */
415 ld r10,HSTATE_SCRATCH0(r13)
416 cmpwi r10,BOOK3S_INTERRUPT_MACHINE_CHECK
417 beq .Lcall_machine_check_common
418
419 cmpwi r10,BOOK3S_INTERRUPT_SYSTEM_RESET
420 beq .Lcall_system_reset_common
421
422 b .
423
424.Lcall_machine_check_common:
425 b machine_check_common
426
427.Lcall_system_reset_common:
428 b system_reset_common
429#endif
430

source code of linux/arch/powerpc/kvm/book3s_64_entry.S