1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 *
4 * Derived from book3s_hv_rmhandlers.S, which is:
5 *
6 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
7 */
8
9#include <linux/export.h>
10#include <asm/reg.h>
11#include <asm/ppc_asm.h>
12#include <asm/asm-offsets.h>
13#include <asm/tm.h>
14#include <asm/cputable.h>
15
16#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
17#define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
18
19/*
20 * Save transactional state and TM-related registers.
21 * Called with:
22 * - r3 pointing to the vcpu struct
23 * - r4 containing the MSR with current TS bits:
24 * (For HV KVM, it is VCPU_MSR ; For PR KVM, it is host MSR).
25 * - r5 containing a flag indicating that non-volatile registers
26 * must be preserved.
27 * If r5 == 0, this can modify all checkpointed registers, but
28 * restores r1, r2 before exit. If r5 != 0, this restores the
29 * MSR TM/FP/VEC/VSX bits to their state on entry.
30 */
31_GLOBAL(__kvmppc_save_tm)
32 mflr r0
33 std r0, PPC_LR_STKOFF(r1)
34 stdu r1, -SWITCH_FRAME_SIZE(r1)
35
36 mr r9, r3
37 cmpdi cr7, r5, 0
38
39 /* Turn on TM. */
40 mfmsr r8
41 mr r10, r8
42 li r0, 1
43 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
44 ori r8, r8, MSR_FP
45 oris r8, r8, (MSR_VEC | MSR_VSX)@h
46 mtmsrd r8
47
48 rldicl. r4, r4, 64 - MSR_TS_S_LG, 62
49 beq 1f /* TM not active in guest. */
50
51 std r1, HSTATE_SCRATCH2(r13)
52 std r3, HSTATE_SCRATCH1(r13)
53
54 /* Save CR on the stack - even if r5 == 0 we need to get cr7 back. */
55 mfcr r6
56 SAVE_GPR(6, r1)
57
58 /* Save DSCR so we can restore it to avoid running with user value */
59 mfspr r7, SPRN_DSCR
60 SAVE_GPR(7, r1)
61
62 /*
63 * We are going to do treclaim., which will modify all checkpointed
64 * registers. Save the non-volatile registers on the stack if
65 * preservation of non-volatile state has been requested.
66 */
67 beq cr7, 3f
68 SAVE_NVGPRS(r1)
69
70 /* MSR[TS] will be 0 (non-transactional) once we do treclaim. */
71 li r0, 0
72 rldimi r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
73 SAVE_GPR(10, r1) /* final MSR value */
743:
75#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
76BEGIN_FTR_SECTION
77 /* Emulation of the treclaim instruction needs TEXASR before treclaim */
78 mfspr r6, SPRN_TEXASR
79 std r6, VCPU_ORIG_TEXASR(r3)
80END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST)
81#endif
82
83 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
84 li r5, 0
85 mtmsrd r5, 1
86
87 li r3, TM_CAUSE_KVM_RESCHED
88
89 /* All GPRs are volatile at this point. */
90 TRECLAIM(R3)
91
92 /* Temporarily store r13 and r9 so we have some regs to play with */
93 SET_SCRATCH0(r13)
94 GET_PACA(r13)
95 std r9, PACATMSCRATCH(r13)
96 ld r9, HSTATE_SCRATCH1(r13)
97
98 /* Save away PPR soon so we don't run with user value. */
99 std r0, VCPU_GPRS_TM(0)(r9)
100 mfspr r0, SPRN_PPR
101 HMT_MEDIUM
102
103 /* Reload stack pointer. */
104 std r1, VCPU_GPRS_TM(1)(r9)
105 ld r1, HSTATE_SCRATCH2(r13)
106
107 /* Set MSR RI now we have r1 and r13 back. */
108 std r2, VCPU_GPRS_TM(2)(r9)
109 li r2, MSR_RI
110 mtmsrd r2, 1
111
112 /* Reload TOC pointer. */
113 LOAD_PACA_TOC()
114
115 /* Save all but r0-r2, r9 & r13 */
116 reg = 3
117 .rept 29
118 .if (reg != 9) && (reg != 13)
119 std reg, VCPU_GPRS_TM(reg)(r9)
120 .endif
121 reg = reg + 1
122 .endr
123 /* ... now save r13 */
124 GET_SCRATCH0(r4)
125 std r4, VCPU_GPRS_TM(13)(r9)
126 /* ... and save r9 */
127 ld r4, PACATMSCRATCH(r13)
128 std r4, VCPU_GPRS_TM(9)(r9)
129
130 /* Restore host DSCR and CR values, after saving guest values */
131 mfcr r6
132 mfspr r7, SPRN_DSCR
133 stw r6, VCPU_CR_TM(r9)
134 std r7, VCPU_DSCR_TM(r9)
135 REST_GPR(6, r1)
136 REST_GPR(7, r1)
137 mtcr r6
138 mtspr SPRN_DSCR, r7
139
140 /* Save away checkpointed SPRs. */
141 std r0, VCPU_PPR_TM(r9)
142 mflr r5
143 mfctr r7
144 mfspr r8, SPRN_AMR
145 mfspr r10, SPRN_TAR
146 mfxer r11
147 std r5, VCPU_LR_TM(r9)
148 std r7, VCPU_CTR_TM(r9)
149 std r8, VCPU_AMR_TM(r9)
150 std r10, VCPU_TAR_TM(r9)
151 std r11, VCPU_XER_TM(r9)
152
153 /* Save FP/VSX. */
154 addi r3, r9, VCPU_FPRS_TM
155 bl store_fp_state
156 addi r3, r9, VCPU_VRS_TM
157 bl store_vr_state
158 mfspr r6, SPRN_VRSAVE
159 stw r6, VCPU_VRSAVE_TM(r9)
160
161 /* Restore non-volatile registers if requested to */
162 beq cr7, 1f
163 REST_NVGPRS(r1)
164 REST_GPR(10, r1)
1651:
166 /*
167 * We need to save these SPRs after the treclaim so that the software
168 * error code is recorded correctly in the TEXASR. Also the user may
169 * change these outside of a transaction, so they must always be
170 * context switched.
171 */
172 mfspr r7, SPRN_TEXASR
173 std r7, VCPU_TEXASR(r9)
174 mfspr r5, SPRN_TFHAR
175 mfspr r6, SPRN_TFIAR
176 std r5, VCPU_TFHAR(r9)
177 std r6, VCPU_TFIAR(r9)
178
179 /* Restore MSR state if requested */
180 beq cr7, 2f
181 mtmsrd r10, 0
1822:
183 addi r1, r1, SWITCH_FRAME_SIZE
184 ld r0, PPC_LR_STKOFF(r1)
185 mtlr r0
186 blr
187
188/*
189 * _kvmppc_save_tm_pr() is a wrapper around __kvmppc_save_tm(), so that it can
190 * be invoked from C function by PR KVM only.
191 */
192_GLOBAL(_kvmppc_save_tm_pr)
193 mflr r0
194 std r0, PPC_LR_STKOFF(r1)
195 stdu r1, -PPC_MIN_STKFRM(r1)
196
197 mfspr r8, SPRN_TAR
198 std r8, PPC_MIN_STKFRM-8(r1)
199
200 li r5, 1 /* preserve non-volatile registers */
201 bl __kvmppc_save_tm
202
203 ld r8, PPC_MIN_STKFRM-8(r1)
204 mtspr SPRN_TAR, r8
205
206 addi r1, r1, PPC_MIN_STKFRM
207 ld r0, PPC_LR_STKOFF(r1)
208 mtlr r0
209 blr
210
211EXPORT_SYMBOL_GPL(_kvmppc_save_tm_pr);
212
213/*
214 * Restore transactional state and TM-related registers.
215 * Called with:
216 * - r3 pointing to the vcpu struct.
217 * - r4 is the guest MSR with desired TS bits:
218 * For HV KVM, it is VCPU_MSR
219 * For PR KVM, it is provided by caller
220 * - r5 containing a flag indicating that non-volatile registers
221 * must be preserved.
222 * If r5 == 0, this potentially modifies all checkpointed registers, but
223 * restores r1, r2 from the PACA before exit.
224 * If r5 != 0, this restores the MSR TM/FP/VEC/VSX bits to their state on entry.
225 */
226_GLOBAL(__kvmppc_restore_tm)
227 mflr r0
228 std r0, PPC_LR_STKOFF(r1)
229
230 cmpdi cr7, r5, 0
231
232 /* Turn on TM/FP/VSX/VMX so we can restore them. */
233 mfmsr r5
234 mr r10, r5
235 li r6, MSR_TM >> 32
236 sldi r6, r6, 32
237 or r5, r5, r6
238 ori r5, r5, MSR_FP
239 oris r5, r5, (MSR_VEC | MSR_VSX)@h
240 mtmsrd r5
241
242 /*
243 * The user may change these outside of a transaction, so they must
244 * always be context switched.
245 */
246 ld r5, VCPU_TFHAR(r3)
247 ld r6, VCPU_TFIAR(r3)
248 ld r7, VCPU_TEXASR(r3)
249 mtspr SPRN_TFHAR, r5
250 mtspr SPRN_TFIAR, r6
251 mtspr SPRN_TEXASR, r7
252
253 mr r5, r4
254 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
255 beq 9f /* TM not active in guest */
256
257 /* Make sure the failure summary is set, otherwise we'll program check
258 * when we trechkpt. It's possible that this might have been not set
259 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
260 * host.
261 */
262 oris r7, r7, (TEXASR_FS)@h
263 mtspr SPRN_TEXASR, r7
264
265 /*
266 * Make a stack frame and save non-volatile registers if requested.
267 */
268 stdu r1, -SWITCH_FRAME_SIZE(r1)
269 std r1, HSTATE_SCRATCH2(r13)
270
271 mfcr r6
272 mfspr r7, SPRN_DSCR
273 SAVE_GPR(2, r1)
274 SAVE_GPR(6, r1)
275 SAVE_GPR(7, r1)
276
277 beq cr7, 4f
278 SAVE_NVGPRS(r1)
279
280 /* MSR[TS] will be 1 (suspended) once we do trechkpt */
281 li r0, 1
282 rldimi r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
283 SAVE_GPR(10, r1) /* final MSR value */
2844:
285 /*
286 * We need to load up the checkpointed state for the guest.
287 * We need to do this early as it will blow away any GPRs, VSRs and
288 * some SPRs.
289 */
290
291 mr r31, r3
292 addi r3, r31, VCPU_FPRS_TM
293 bl load_fp_state
294 addi r3, r31, VCPU_VRS_TM
295 bl load_vr_state
296 mr r3, r31
297 lwz r7, VCPU_VRSAVE_TM(r3)
298 mtspr SPRN_VRSAVE, r7
299
300 ld r5, VCPU_LR_TM(r3)
301 lwz r6, VCPU_CR_TM(r3)
302 ld r7, VCPU_CTR_TM(r3)
303 ld r8, VCPU_AMR_TM(r3)
304 ld r9, VCPU_TAR_TM(r3)
305 ld r10, VCPU_XER_TM(r3)
306 mtlr r5
307 mtcr r6
308 mtctr r7
309 mtspr SPRN_AMR, r8
310 mtspr SPRN_TAR, r9
311 mtxer r10
312
313 /*
314 * Load up PPR and DSCR values but don't put them in the actual SPRs
315 * till the last moment to avoid running with userspace PPR and DSCR for
316 * too long.
317 */
318 ld r29, VCPU_DSCR_TM(r3)
319 ld r30, VCPU_PPR_TM(r3)
320
321 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
322 li r5, 0
323 mtmsrd r5, 1
324
325 /* Load GPRs r0-r28 */
326 reg = 0
327 .rept 29
328 ld reg, VCPU_GPRS_TM(reg)(r31)
329 reg = reg + 1
330 .endr
331
332 mtspr SPRN_DSCR, r29
333 mtspr SPRN_PPR, r30
334
335 /* Load final GPRs */
336 ld 29, VCPU_GPRS_TM(29)(r31)
337 ld 30, VCPU_GPRS_TM(30)(r31)
338 ld 31, VCPU_GPRS_TM(31)(r31)
339
340 /* TM checkpointed state is now setup. All GPRs are now volatile. */
341 TRECHKPT
342
343 /* Now let's get back the state we need. */
344 HMT_MEDIUM
345 GET_PACA(r13)
346 ld r1, HSTATE_SCRATCH2(r13)
347 REST_GPR(7, r1)
348 mtspr SPRN_DSCR, r7
349
350 /* Set the MSR RI since we have our registers back. */
351 li r5, MSR_RI
352 mtmsrd r5, 1
353
354 /* Restore TOC pointer and CR */
355 REST_GPR(2, r1)
356 REST_GPR(6, r1)
357 mtcr r6
358
359 /* Restore non-volatile registers if requested to. */
360 beq cr7, 5f
361 REST_GPR(10, r1)
362 REST_NVGPRS(r1)
363
3645: addi r1, r1, SWITCH_FRAME_SIZE
365 ld r0, PPC_LR_STKOFF(r1)
366 mtlr r0
367
3689: /* Restore MSR bits if requested */
369 beqlr cr7
370 mtmsrd r10, 0
371 blr
372
373/*
374 * _kvmppc_restore_tm_pr() is a wrapper around __kvmppc_restore_tm(), so that it
375 * can be invoked from C function by PR KVM only.
376 */
377_GLOBAL(_kvmppc_restore_tm_pr)
378 mflr r0
379 std r0, PPC_LR_STKOFF(r1)
380 stdu r1, -PPC_MIN_STKFRM(r1)
381
382 /* save TAR so that it can be recovered later */
383 mfspr r8, SPRN_TAR
384 std r8, PPC_MIN_STKFRM-8(r1)
385
386 li r5, 1
387 bl __kvmppc_restore_tm
388
389 ld r8, PPC_MIN_STKFRM-8(r1)
390 mtspr SPRN_TAR, r8
391
392 addi r1, r1, PPC_MIN_STKFRM
393 ld r0, PPC_LR_STKOFF(r1)
394 mtlr r0
395 blr
396
397EXPORT_SYMBOL_GPL(_kvmppc_restore_tm_pr);
398#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
399

source code of linux/arch/powerpc/kvm/tm.S