1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * Hypervisor stub |
4 | * |
5 | * Copyright (C) 2012 ARM Ltd. |
6 | * Author: Marc Zyngier <marc.zyngier@arm.com> |
7 | */ |
8 | |
9 | #include <linux/init.h> |
10 | #include <linux/linkage.h> |
11 | |
12 | #include <asm/assembler.h> |
13 | #include <asm/el2_setup.h> |
14 | #include <asm/kvm_arm.h> |
15 | #include <asm/kvm_asm.h> |
16 | #include <asm/ptrace.h> |
17 | #include <asm/virt.h> |
18 | |
19 | .text |
20 | .pushsection .hyp.text, "ax" |
21 | |
22 | .align 11 |
23 | |
24 | SYM_CODE_START(__hyp_stub_vectors) |
25 | ventry el2_sync_invalid // Synchronous EL2t |
26 | ventry el2_irq_invalid // IRQ EL2t |
27 | ventry el2_fiq_invalid // FIQ EL2t |
28 | ventry el2_error_invalid // Error EL2t |
29 | |
30 | ventry elx_sync // Synchronous EL2h |
31 | ventry el2_irq_invalid // IRQ EL2h |
32 | ventry el2_fiq_invalid // FIQ EL2h |
33 | ventry el2_error_invalid // Error EL2h |
34 | |
35 | ventry elx_sync // Synchronous 64-bit EL1 |
36 | ventry el1_irq_invalid // IRQ 64-bit EL1 |
37 | ventry el1_fiq_invalid // FIQ 64-bit EL1 |
38 | ventry el1_error_invalid // Error 64-bit EL1 |
39 | |
40 | ventry el1_sync_invalid // Synchronous 32-bit EL1 |
41 | ventry el1_irq_invalid // IRQ 32-bit EL1 |
42 | ventry el1_fiq_invalid // FIQ 32-bit EL1 |
43 | ventry el1_error_invalid // Error 32-bit EL1 |
44 | SYM_CODE_END(__hyp_stub_vectors) |
45 | |
46 | .align 11 |
47 | |
48 | SYM_CODE_START_LOCAL(elx_sync) |
49 | cmp x0, #HVC_SET_VECTORS |
50 | b.ne 1f |
51 | msr vbar_el2, x1 |
52 | b 9f |
53 | |
54 | 1: cmp x0, #HVC_FINALISE_EL2 |
55 | b.eq __finalise_el2 |
56 | |
57 | 2: cmp x0, #HVC_SOFT_RESTART |
58 | b.ne 3f |
59 | mov x0, x2 |
60 | mov x2, x4 |
61 | mov x4, x1 |
62 | mov x1, x3 |
63 | br x4 // no return |
64 | |
65 | 3: cmp x0, #HVC_RESET_VECTORS |
66 | beq 9f // Nothing to reset! |
67 | |
68 | /* Someone called kvm_call_hyp() against the hyp-stub... */ |
69 | mov_q x0, HVC_STUB_ERR |
70 | eret |
71 | |
72 | 9: mov x0, xzr |
73 | eret |
74 | SYM_CODE_END(elx_sync) |
75 | |
76 | SYM_CODE_START_LOCAL(__finalise_el2) |
77 | finalise_el2_state |
78 | |
79 | // nVHE? No way! Give me the real thing! |
80 | // Sanity check: MMU *must* be off |
81 | mrs x1, sctlr_el2 |
82 | tbnz x1, #0, 1f |
83 | |
84 | // Needs to be VHE capable, obviously |
85 | check_override id_aa64mmfr1 ID_AA64MMFR1_EL1_VH_SHIFT 0f 1f x1 x2 |
86 | |
87 | 0: // Check whether we only want the hypervisor to run VHE, not the kernel |
88 | adr_l x1, arm64_sw_feature_override |
89 | ldr x2, [x1, FTR_OVR_VAL_OFFSET] |
90 | ldr x1, [x1, FTR_OVR_MASK_OFFSET] |
91 | and x2, x2, x1 |
92 | ubfx x2, x2, #ARM64_SW_FEATURE_OVERRIDE_HVHE, #4 |
93 | cbz x2, 2f |
94 | |
95 | 1: mov_q x0, HVC_STUB_ERR |
96 | eret |
97 | 2: |
98 | // Engage the VHE magic! |
99 | mov_q x0, HCR_HOST_VHE_FLAGS |
100 | msr hcr_el2, x0 |
101 | isb |
102 | |
103 | // Use the EL1 allocated stack, per-cpu offset |
104 | mrs x0, sp_el1 |
105 | mov sp, x0 |
106 | mrs x0, tpidr_el1 |
107 | msr tpidr_el2, x0 |
108 | |
109 | // FP configuration, vectors |
110 | mrs_s x0, SYS_CPACR_EL12 |
111 | msr cpacr_el1, x0 |
112 | mrs_s x0, SYS_VBAR_EL12 |
113 | msr vbar_el1, x0 |
114 | |
115 | // Use EL2 translations for SPE & TRBE and disable access from EL1 |
116 | mrs x0, mdcr_el2 |
117 | bic x0, x0, #(MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT) |
118 | bic x0, x0, #(MDCR_EL2_E2TB_MASK << MDCR_EL2_E2TB_SHIFT) |
119 | msr mdcr_el2, x0 |
120 | |
121 | // Transfer the MM state from EL1 to EL2 |
122 | mrs_s x0, SYS_TCR_EL12 |
123 | msr tcr_el1, x0 |
124 | mrs_s x0, SYS_TTBR0_EL12 |
125 | msr ttbr0_el1, x0 |
126 | mrs_s x0, SYS_TTBR1_EL12 |
127 | msr ttbr1_el1, x0 |
128 | mrs_s x0, SYS_MAIR_EL12 |
129 | msr mair_el1, x0 |
130 | mrs x1, REG_ID_AA64MMFR3_EL1 |
131 | ubfx x1, x1, #ID_AA64MMFR3_EL1_TCRX_SHIFT, #4 |
132 | cbz x1, .Lskip_tcr2 |
133 | mrs x0, REG_TCR2_EL12 |
134 | msr REG_TCR2_EL1, x0 |
135 | |
136 | // Transfer permission indirection state |
137 | mrs x1, REG_ID_AA64MMFR3_EL1 |
138 | ubfx x1, x1, #ID_AA64MMFR3_EL1_S1PIE_SHIFT, #4 |
139 | cbz x1, .Lskip_indirection |
140 | mrs x0, REG_PIRE0_EL12 |
141 | msr REG_PIRE0_EL1, x0 |
142 | mrs x0, REG_PIR_EL12 |
143 | msr REG_PIR_EL1, x0 |
144 | |
145 | .Lskip_indirection: |
146 | .Lskip_tcr2: |
147 | |
148 | isb |
149 | |
150 | // Hack the exception return to stay at EL2 |
151 | mrs x0, spsr_el1 |
152 | and x0, x0, #~PSR_MODE_MASK |
153 | mov x1, #PSR_MODE_EL2h |
154 | orr x0, x0, x1 |
155 | msr spsr_el1, x0 |
156 | |
157 | b enter_vhe |
158 | SYM_CODE_END(__finalise_el2) |
159 | |
160 | // At the point where we reach enter_vhe(), we run with |
161 | // the MMU off (which is enforced by __finalise_el2()). |
162 | // We thus need to be in the idmap, or everything will |
163 | // explode when enabling the MMU. |
164 | |
165 | .pushsection .idmap.text, "ax" |
166 | |
167 | SYM_CODE_START_LOCAL(enter_vhe) |
168 | // Invalidate TLBs before enabling the MMU |
169 | tlbi vmalle1 |
170 | dsb nsh |
171 | isb |
172 | |
173 | // Enable the EL2 S1 MMU, as set up from EL1 |
174 | mrs_s x0, SYS_SCTLR_EL12 |
175 | set_sctlr_el1 x0 |
176 | |
177 | // Disable the EL1 S1 MMU for a good measure |
178 | mov_q x0, INIT_SCTLR_EL1_MMU_OFF |
179 | msr_s SYS_SCTLR_EL12, x0 |
180 | |
181 | mov x0, xzr |
182 | |
183 | eret |
184 | SYM_CODE_END(enter_vhe) |
185 | |
186 | .popsection |
187 | |
188 | .macro invalid_vector label |
189 | SYM_CODE_START_LOCAL(\label) |
190 | b \label |
191 | SYM_CODE_END(\label) |
192 | .endm |
193 | |
194 | invalid_vector el2_sync_invalid |
195 | invalid_vector el2_irq_invalid |
196 | invalid_vector el2_fiq_invalid |
197 | invalid_vector el2_error_invalid |
198 | invalid_vector el1_sync_invalid |
199 | invalid_vector el1_irq_invalid |
200 | invalid_vector el1_fiq_invalid |
201 | invalid_vector el1_error_invalid |
202 | |
203 | .popsection |
204 | |
205 | /* |
206 | * __hyp_set_vectors: Call this after boot to set the initial hypervisor |
207 | * vectors as part of hypervisor installation. On an SMP system, this should |
208 | * be called on each CPU. |
209 | * |
210 | * x0 must be the physical address of the new vector table, and must be |
211 | * 2KB aligned. |
212 | * |
213 | * Before calling this, you must check that the stub hypervisor is installed |
214 | * everywhere, by waiting for any secondary CPUs to be brought up and then |
215 | * checking that is_hyp_mode_available() is true. |
216 | * |
217 | * If not, there is a pre-existing hypervisor, some CPUs failed to boot, or |
218 | * something else went wrong... in such cases, trying to install a new |
219 | * hypervisor is unlikely to work as desired. |
220 | * |
221 | * When you call into your shiny new hypervisor, sp_el2 will contain junk, |
222 | * so you will need to set that to something sensible at the new hypervisor's |
223 | * initialisation entry point. |
224 | */ |
225 | |
226 | SYM_FUNC_START(__hyp_set_vectors) |
227 | mov x1, x0 |
228 | mov x0, #HVC_SET_VECTORS |
229 | hvc #0 |
230 | ret |
231 | SYM_FUNC_END(__hyp_set_vectors) |
232 | |
233 | SYM_FUNC_START(__hyp_reset_vectors) |
234 | mov x0, #HVC_RESET_VECTORS |
235 | hvc #0 |
236 | ret |
237 | SYM_FUNC_END(__hyp_reset_vectors) |
238 | |
239 | /* |
240 | * Entry point to finalise EL2 and switch to VHE if deemed capable |
241 | * |
242 | * w0: boot mode, as returned by init_kernel_el() |
243 | */ |
244 | SYM_FUNC_START(finalise_el2) |
245 | // Need to have booted at EL2 |
246 | cmp w0, #BOOT_CPU_MODE_EL2 |
247 | b.ne 1f |
248 | |
249 | // and still be at EL1 |
250 | mrs x0, CurrentEL |
251 | cmp x0, #CurrentEL_EL1 |
252 | b.ne 1f |
253 | |
254 | mov x0, #HVC_FINALISE_EL2 |
255 | hvc #0 |
256 | 1: |
257 | ret |
258 | SYM_FUNC_END(finalise_el2) |
259 | |