1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012-2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <hyp/sysreg-sr.h>
8
9#include <linux/compiler.h>
10#include <linux/kvm_host.h>
11
12#include <asm/kprobes.h>
13#include <asm/kvm_asm.h>
14#include <asm/kvm_emulate.h>
15#include <asm/kvm_hyp.h>
16#include <asm/kvm_nested.h>
17
18static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu)
19{
20 /* These registers are common with EL1 */
21 __vcpu_assign_sys_reg(vcpu, PAR_EL1, read_sysreg(par_el1));
22 __vcpu_assign_sys_reg(vcpu, TPIDR_EL1, read_sysreg(tpidr_el1));
23
24 __vcpu_assign_sys_reg(vcpu, ESR_EL2, read_sysreg_el1(SYS_ESR));
25 __vcpu_assign_sys_reg(vcpu, AFSR0_EL2, read_sysreg_el1(SYS_AFSR0));
26 __vcpu_assign_sys_reg(vcpu, AFSR1_EL2, read_sysreg_el1(SYS_AFSR1));
27 __vcpu_assign_sys_reg(vcpu, FAR_EL2, read_sysreg_el1(SYS_FAR));
28 __vcpu_assign_sys_reg(vcpu, MAIR_EL2, read_sysreg_el1(SYS_MAIR));
29 __vcpu_assign_sys_reg(vcpu, VBAR_EL2, read_sysreg_el1(SYS_VBAR));
30 __vcpu_assign_sys_reg(vcpu, CONTEXTIDR_EL2, read_sysreg_el1(SYS_CONTEXTIDR));
31 __vcpu_assign_sys_reg(vcpu, AMAIR_EL2, read_sysreg_el1(SYS_AMAIR));
32
33 /*
34 * In VHE mode those registers are compatible between EL1 and EL2,
35 * and the guest uses the _EL1 versions on the CPU naturally.
36 * So we save them into their _EL2 versions here.
37 * For nVHE mode we trap accesses to those registers, so our
38 * _EL2 copy in sys_regs[] is always up-to-date and we don't need
39 * to save anything here.
40 */
41 if (vcpu_el2_e2h_is_set(vcpu)) {
42 u64 val;
43
44 /*
45 * We don't save CPTR_EL2, as accesses to CPACR_EL1
46 * are always trapped, ensuring that the in-memory
47 * copy is always up-to-date. A small blessing...
48 */
49 __vcpu_assign_sys_reg(vcpu, SCTLR_EL2, read_sysreg_el1(SYS_SCTLR));
50 __vcpu_assign_sys_reg(vcpu, TTBR0_EL2, read_sysreg_el1(SYS_TTBR0));
51 __vcpu_assign_sys_reg(vcpu, TTBR1_EL2, read_sysreg_el1(SYS_TTBR1));
52 __vcpu_assign_sys_reg(vcpu, TCR_EL2, read_sysreg_el1(SYS_TCR));
53
54 if (ctxt_has_tcrx(&vcpu->arch.ctxt)) {
55 __vcpu_assign_sys_reg(vcpu, TCR2_EL2, read_sysreg_el1(SYS_TCR2));
56
57 if (ctxt_has_s1pie(&vcpu->arch.ctxt)) {
58 __vcpu_assign_sys_reg(vcpu, PIRE0_EL2, read_sysreg_el1(SYS_PIRE0));
59 __vcpu_assign_sys_reg(vcpu, PIR_EL2, read_sysreg_el1(SYS_PIR));
60 }
61
62 if (ctxt_has_s1poe(&vcpu->arch.ctxt))
63 __vcpu_assign_sys_reg(vcpu, POR_EL2, read_sysreg_el1(SYS_POR));
64 }
65
66 /*
67 * The EL1 view of CNTKCTL_EL1 has a bunch of RES0 bits where
68 * the interesting CNTHCTL_EL2 bits live. So preserve these
69 * bits when reading back the guest-visible value.
70 */
71 val = read_sysreg_el1(SYS_CNTKCTL);
72 val &= CNTKCTL_VALID_BITS;
73 __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, &=, ~CNTKCTL_VALID_BITS);
74 __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, |=, val);
75 }
76
77 __vcpu_assign_sys_reg(vcpu, SP_EL2, read_sysreg(sp_el1));
78 __vcpu_assign_sys_reg(vcpu, ELR_EL2, read_sysreg_el1(SYS_ELR));
79 __vcpu_assign_sys_reg(vcpu, SPSR_EL2, read_sysreg_el1(SYS_SPSR));
80
81 if (ctxt_has_sctlr2(&vcpu->arch.ctxt))
82 __vcpu_assign_sys_reg(vcpu, SCTLR2_EL2, read_sysreg_el1(SYS_SCTLR2));
83}
84
85static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu)
86{
87 u64 val;
88
89 /* These registers are common with EL1 */
90 write_sysreg(__vcpu_sys_reg(vcpu, PAR_EL1), par_el1);
91 write_sysreg(__vcpu_sys_reg(vcpu, TPIDR_EL1), tpidr_el1);
92
93 write_sysreg(ctxt_midr_el1(&vcpu->arch.ctxt), vpidr_el2);
94 write_sysreg(__vcpu_sys_reg(vcpu, MPIDR_EL1), vmpidr_el2);
95 write_sysreg_el1(__vcpu_sys_reg(vcpu, MAIR_EL2), SYS_MAIR);
96 write_sysreg_el1(__vcpu_sys_reg(vcpu, VBAR_EL2), SYS_VBAR);
97 write_sysreg_el1(__vcpu_sys_reg(vcpu, CONTEXTIDR_EL2), SYS_CONTEXTIDR);
98 write_sysreg_el1(__vcpu_sys_reg(vcpu, AMAIR_EL2), SYS_AMAIR);
99
100 if (vcpu_el2_e2h_is_set(vcpu)) {
101 /*
102 * In VHE mode those registers are compatible between
103 * EL1 and EL2.
104 */
105 write_sysreg_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2), SYS_SCTLR);
106 write_sysreg_el1(__vcpu_sys_reg(vcpu, CPTR_EL2), SYS_CPACR);
107 write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2), SYS_TTBR0);
108 write_sysreg_el1(__vcpu_sys_reg(vcpu, TTBR1_EL2), SYS_TTBR1);
109 write_sysreg_el1(__vcpu_sys_reg(vcpu, TCR_EL2), SYS_TCR);
110 write_sysreg_el1(__vcpu_sys_reg(vcpu, CNTHCTL_EL2), SYS_CNTKCTL);
111 } else {
112 /*
113 * CNTHCTL_EL2 only affects EL1 when running nVHE, so
114 * no need to restore it.
115 */
116 val = translate_sctlr_el2_to_sctlr_el1(__vcpu_sys_reg(vcpu, SCTLR_EL2));
117 write_sysreg_el1(val, SYS_SCTLR);
118 val = translate_cptr_el2_to_cpacr_el1(__vcpu_sys_reg(vcpu, CPTR_EL2));
119 write_sysreg_el1(val, SYS_CPACR);
120 val = translate_ttbr0_el2_to_ttbr0_el1(__vcpu_sys_reg(vcpu, TTBR0_EL2));
121 write_sysreg_el1(val, SYS_TTBR0);
122 val = translate_tcr_el2_to_tcr_el1(__vcpu_sys_reg(vcpu, TCR_EL2));
123 write_sysreg_el1(val, SYS_TCR);
124 }
125
126 if (ctxt_has_tcrx(&vcpu->arch.ctxt)) {
127 write_sysreg_el1(__vcpu_sys_reg(vcpu, TCR2_EL2), SYS_TCR2);
128
129 if (ctxt_has_s1pie(&vcpu->arch.ctxt)) {
130 write_sysreg_el1(__vcpu_sys_reg(vcpu, PIR_EL2), SYS_PIR);
131 write_sysreg_el1(__vcpu_sys_reg(vcpu, PIRE0_EL2), SYS_PIRE0);
132 }
133
134 if (ctxt_has_s1poe(&vcpu->arch.ctxt))
135 write_sysreg_el1(__vcpu_sys_reg(vcpu, POR_EL2), SYS_POR);
136 }
137
138 write_sysreg_el1(__vcpu_sys_reg(vcpu, ESR_EL2), SYS_ESR);
139 write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR0_EL2), SYS_AFSR0);
140 write_sysreg_el1(__vcpu_sys_reg(vcpu, AFSR1_EL2), SYS_AFSR1);
141 write_sysreg_el1(__vcpu_sys_reg(vcpu, FAR_EL2), SYS_FAR);
142 write_sysreg(__vcpu_sys_reg(vcpu, SP_EL2), sp_el1);
143 write_sysreg_el1(__vcpu_sys_reg(vcpu, ELR_EL2), SYS_ELR);
144 write_sysreg_el1(__vcpu_sys_reg(vcpu, SPSR_EL2), SYS_SPSR);
145
146 if (ctxt_has_sctlr2(&vcpu->arch.ctxt))
147 write_sysreg_el1(__vcpu_sys_reg(vcpu, SCTLR2_EL2), SYS_SCTLR2);
148}
149
150/*
151 * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
152 * pstate, which are handled as part of the el2 return state) on every
153 * switch (sp_el0 is being dealt with in the assembly code).
154 * tpidr_el0 and tpidrro_el0 only need to be switched when going
155 * to host userspace or a different VCPU. EL1 registers only need to be
156 * switched when potentially going to run a different VCPU. The latter two
157 * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put.
158 */
159
160void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
161{
162 __sysreg_save_common_state(ctxt);
163}
164NOKPROBE_SYMBOL(sysreg_save_host_state_vhe);
165
166void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
167{
168 __sysreg_save_common_state(ctxt);
169 __sysreg_save_el2_return_state(ctxt);
170}
171NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe);
172
173void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
174{
175 __sysreg_restore_common_state(ctxt);
176}
177NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe);
178
179void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
180{
181 __sysreg_restore_common_state(ctxt);
182 __sysreg_restore_el2_return_state(ctxt);
183}
184NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe);
185
186/**
187 * __vcpu_load_switch_sysregs - Load guest system registers to the physical CPU
188 *
189 * @vcpu: The VCPU pointer
190 *
191 * Load system registers that do not affect the host's execution, for
192 * example EL1 system registers on a VHE system where the host kernel
193 * runs at EL2. This function is called from KVM's vcpu_load() function
194 * and loading system register state early avoids having to load them on
195 * every entry to the VM.
196 */
197void __vcpu_load_switch_sysregs(struct kvm_vcpu *vcpu)
198{
199 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
200 struct kvm_cpu_context *host_ctxt;
201 u64 midr, mpidr;
202
203 host_ctxt = host_data_ptr(host_ctxt);
204 __sysreg_save_user_state(host_ctxt);
205
206 /*
207 * When running a normal EL1 guest, we only load a new vcpu
208 * after a context switch, which imvolves a DSB, so all
209 * speculative EL1&0 walks will have already completed.
210 * If running NV, the vcpu may transition between vEL1 and
211 * vEL2 without a context switch, so make sure we complete
212 * those walks before loading a new context.
213 */
214 if (vcpu_has_nv(vcpu))
215 dsb(nsh);
216
217 /*
218 * Load guest EL1 and user state
219 *
220 * We must restore the 32-bit state before the sysregs, thanks
221 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
222 */
223 __sysreg32_restore_state(vcpu);
224 __sysreg_restore_user_state(guest_ctxt);
225
226 if (unlikely(is_hyp_ctxt(vcpu))) {
227 __sysreg_restore_vel2_state(vcpu);
228 } else {
229 if (vcpu_has_nv(vcpu)) {
230 /*
231 * As we're restoring a nested guest, set the value
232 * provided by the guest hypervisor.
233 */
234 midr = ctxt_sys_reg(guest_ctxt, VPIDR_EL2);
235 mpidr = ctxt_sys_reg(guest_ctxt, VMPIDR_EL2);
236 } else {
237 midr = ctxt_midr_el1(guest_ctxt);
238 mpidr = ctxt_sys_reg(guest_ctxt, MPIDR_EL1);
239 }
240
241 __sysreg_restore_el1_state(guest_ctxt, midr, mpidr);
242 }
243
244 vcpu_set_flag(vcpu, SYSREGS_ON_CPU);
245}
246
247/**
248 * __vcpu_put_switch_sysregs - Restore host system registers to the physical CPU
249 *
250 * @vcpu: The VCPU pointer
251 *
252 * Save guest system registers that do not affect the host's execution, for
253 * example EL1 system registers on a VHE system where the host kernel
254 * runs at EL2. This function is called from KVM's vcpu_put() function
255 * and deferring saving system register state until we're no longer running the
256 * VCPU avoids having to save them on every exit from the VM.
257 */
258void __vcpu_put_switch_sysregs(struct kvm_vcpu *vcpu)
259{
260 struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
261 struct kvm_cpu_context *host_ctxt;
262
263 host_ctxt = host_data_ptr(host_ctxt);
264
265 if (unlikely(is_hyp_ctxt(vcpu)))
266 __sysreg_save_vel2_state(vcpu);
267 else
268 __sysreg_save_el1_state(guest_ctxt);
269
270 __sysreg_save_user_state(guest_ctxt);
271 __sysreg32_save_state(vcpu);
272
273 /* Restore host user state */
274 __sysreg_restore_user_state(host_ctxt);
275
276 vcpu_clear_flag(vcpu, SYSREGS_ON_CPU);
277}
278

source code of linux/arch/arm64/kvm/hyp/vhe/sysreg-sr.c