1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2017 ARM Ltd.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <linux/kvm_host.h>
8#include <linux/random.h>
9#include <linux/memblock.h>
10#include <asm/alternative.h>
11#include <asm/debug-monitors.h>
12#include <asm/insn.h>
13#include <asm/kvm_mmu.h>
14#include <asm/memory.h>
15
16/*
17 * The LSB of the HYP VA tag
18 */
19static u8 tag_lsb;
20/*
21 * The HYP VA tag value with the region bit
22 */
23static u64 tag_val;
24static u64 va_mask;
25
26/*
27 * Compute HYP VA by using the same computation as kern_hyp_va().
28 */
29static u64 __early_kern_hyp_va(u64 addr)
30{
31 addr &= va_mask;
32 addr |= tag_val << tag_lsb;
33 return addr;
34}
35
36/*
37 * Store a hyp VA <-> PA offset into a EL2-owned variable.
38 */
39static void init_hyp_physvirt_offset(void)
40{
41 u64 kern_va, hyp_va;
42
43 /* Compute the offset from the hyp VA and PA of a random symbol. */
44 kern_va = (u64)lm_alias(__hyp_text_start);
45 hyp_va = __early_kern_hyp_va(addr: kern_va);
46 hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va;
47}
48
49/*
50 * We want to generate a hyp VA with the following format (with V ==
51 * vabits_actual):
52 *
53 * 63 ... V | V-1 | V-2 .. tag_lsb | tag_lsb - 1 .. 0
54 * ---------------------------------------------------------
55 * | 0000000 | hyp_va_msb | random tag | kern linear VA |
56 * |--------- tag_val -----------|----- va_mask ---|
57 *
58 * which does not conflict with the idmap regions.
59 */
60__init void kvm_compute_layout(void)
61{
62 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
63 u64 hyp_va_msb;
64
65 /* Where is my RAM region? */
66 hyp_va_msb = idmap_addr & BIT(vabits_actual - 1);
67 hyp_va_msb ^= BIT(vabits_actual - 1);
68
69 tag_lsb = fls64(x: (u64)phys_to_virt(address: memblock_start_of_DRAM()) ^
70 (u64)(high_memory - 1));
71
72 va_mask = GENMASK_ULL(tag_lsb - 1, 0);
73 tag_val = hyp_va_msb;
74
75 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (vabits_actual - 1)) {
76 /* We have some free bits to insert a random tag. */
77 tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb);
78 }
79 tag_val >>= tag_lsb;
80
81 init_hyp_physvirt_offset();
82}
83
84/*
85 * The .hyp.reloc ELF section contains a list of kimg positions that
86 * contains kimg VAs but will be accessed only in hyp execution context.
87 * Convert them to hyp VAs. See gen-hyprel.c for more details.
88 */
89__init void kvm_apply_hyp_relocations(void)
90{
91 int32_t *rel;
92 int32_t *begin = (int32_t *)__hyp_reloc_begin;
93 int32_t *end = (int32_t *)__hyp_reloc_end;
94
95 for (rel = begin; rel < end; ++rel) {
96 uintptr_t *ptr, kimg_va;
97
98 /*
99 * Each entry contains a 32-bit relative offset from itself
100 * to a kimg VA position.
101 */
102 ptr = (uintptr_t *)lm_alias((char *)rel + *rel);
103
104 /* Read the kimg VA value at the relocation address. */
105 kimg_va = *ptr;
106
107 /* Convert to hyp VA and store back to the relocation address. */
108 *ptr = __early_kern_hyp_va(addr: (uintptr_t)lm_alias(kimg_va));
109 }
110}
111
112static u32 compute_instruction(int n, u32 rd, u32 rn)
113{
114 u32 insn = AARCH64_BREAK_FAULT;
115
116 switch (n) {
117 case 0:
118 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND,
119 AARCH64_INSN_VARIANT_64BIT,
120 rn, rd, va_mask);
121 break;
122
123 case 1:
124 /* ROR is a variant of EXTR with Rm = Rn */
125 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
126 rn, rn, rd,
127 tag_lsb);
128 break;
129
130 case 2:
131 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
132 tag_val & GENMASK(11, 0),
133 AARCH64_INSN_VARIANT_64BIT,
134 AARCH64_INSN_ADSB_ADD);
135 break;
136
137 case 3:
138 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
139 tag_val & GENMASK(23, 12),
140 AARCH64_INSN_VARIANT_64BIT,
141 AARCH64_INSN_ADSB_ADD);
142 break;
143
144 case 4:
145 /* ROR is a variant of EXTR with Rm = Rn */
146 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
147 rn, rn, rd, 64 - tag_lsb);
148 break;
149 }
150
151 return insn;
152}
153
154void __init kvm_update_va_mask(struct alt_instr *alt,
155 __le32 *origptr, __le32 *updptr, int nr_inst)
156{
157 int i;
158
159 BUG_ON(nr_inst != 5);
160
161 for (i = 0; i < nr_inst; i++) {
162 u32 rd, rn, insn, oinsn;
163
164 /*
165 * VHE doesn't need any address translation, let's NOP
166 * everything.
167 *
168 * Alternatively, if the tag is zero (because the layout
169 * dictates it and we don't have any spare bits in the
170 * address), NOP everything after masking the kernel VA.
171 */
172 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) {
173 updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
174 continue;
175 }
176
177 oinsn = le32_to_cpu(origptr[i]);
178 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
179 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
180
181 insn = compute_instruction(n: i, rd, rn);
182 BUG_ON(insn == AARCH64_BREAK_FAULT);
183
184 updptr[i] = cpu_to_le32(insn);
185 }
186}
187
188void kvm_patch_vector_branch(struct alt_instr *alt,
189 __le32 *origptr, __le32 *updptr, int nr_inst)
190{
191 u64 addr;
192 u32 insn;
193
194 BUG_ON(nr_inst != 4);
195
196 if (!cpus_have_cap(ARM64_SPECTRE_V3A) ||
197 WARN_ON_ONCE(cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)))
198 return;
199
200 /*
201 * Compute HYP VA by using the same computation as kern_hyp_va()
202 */
203 addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
204
205 /* Use PC[10:7] to branch to the same vector in KVM */
206 addr |= ((u64)origptr & GENMASK_ULL(10, 7));
207
208 /*
209 * Branch over the preamble in order to avoid the initial store on
210 * the stack (which we already perform in the hardening vectors).
211 */
212 addr += KVM_VECTOR_PREAMBLE;
213
214 /* movz x0, #(addr & 0xffff) */
215 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
216 (u16)addr,
217 0,
218 AARCH64_INSN_VARIANT_64BIT,
219 AARCH64_INSN_MOVEWIDE_ZERO);
220 *updptr++ = cpu_to_le32(insn);
221
222 /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */
223 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
224 (u16)(addr >> 16),
225 16,
226 AARCH64_INSN_VARIANT_64BIT,
227 AARCH64_INSN_MOVEWIDE_KEEP);
228 *updptr++ = cpu_to_le32(insn);
229
230 /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */
231 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
232 (u16)(addr >> 32),
233 32,
234 AARCH64_INSN_VARIANT_64BIT,
235 AARCH64_INSN_MOVEWIDE_KEEP);
236 *updptr++ = cpu_to_le32(insn);
237
238 /* br x0 */
239 insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0,
240 AARCH64_INSN_BRANCH_NOLINK);
241 *updptr++ = cpu_to_le32(insn);
242}
243
244static void generate_mov_q(u64 val, __le32 *origptr, __le32 *updptr, int nr_inst)
245{
246 u32 insn, oinsn, rd;
247
248 BUG_ON(nr_inst != 4);
249
250 /* Compute target register */
251 oinsn = le32_to_cpu(*origptr);
252 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
253
254 /* movz rd, #(val & 0xffff) */
255 insn = aarch64_insn_gen_movewide(rd,
256 (u16)val,
257 0,
258 AARCH64_INSN_VARIANT_64BIT,
259 AARCH64_INSN_MOVEWIDE_ZERO);
260 *updptr++ = cpu_to_le32(insn);
261
262 /* movk rd, #((val >> 16) & 0xffff), lsl #16 */
263 insn = aarch64_insn_gen_movewide(rd,
264 (u16)(val >> 16),
265 16,
266 AARCH64_INSN_VARIANT_64BIT,
267 AARCH64_INSN_MOVEWIDE_KEEP);
268 *updptr++ = cpu_to_le32(insn);
269
270 /* movk rd, #((val >> 32) & 0xffff), lsl #32 */
271 insn = aarch64_insn_gen_movewide(rd,
272 (u16)(val >> 32),
273 32,
274 AARCH64_INSN_VARIANT_64BIT,
275 AARCH64_INSN_MOVEWIDE_KEEP);
276 *updptr++ = cpu_to_le32(insn);
277
278 /* movk rd, #((val >> 48) & 0xffff), lsl #48 */
279 insn = aarch64_insn_gen_movewide(rd,
280 (u16)(val >> 48),
281 48,
282 AARCH64_INSN_VARIANT_64BIT,
283 AARCH64_INSN_MOVEWIDE_KEEP);
284 *updptr++ = cpu_to_le32(insn);
285}
286
287void kvm_get_kimage_voffset(struct alt_instr *alt,
288 __le32 *origptr, __le32 *updptr, int nr_inst)
289{
290 generate_mov_q(kimage_voffset, origptr, updptr, nr_inst);
291}
292
293void kvm_compute_final_ctr_el0(struct alt_instr *alt,
294 __le32 *origptr, __le32 *updptr, int nr_inst)
295{
296 generate_mov_q(read_sanitised_ftr_reg(SYS_CTR_EL0),
297 origptr, updptr, nr_inst);
298}
299
300void kvm_pan_patch_el2_entry(struct alt_instr *alt,
301 __le32 *origptr, __le32 *updptr, int nr_inst)
302{
303 /*
304 * If we're running at EL1 without hVHE, then SCTLR_EL2.SPAN means
305 * nothing to us (it is RES1), and we don't need to set PSTATE.PAN
306 * to anything useful.
307 */
308 if (!is_kernel_in_hyp_mode() && !cpus_have_cap(ARM64_KVM_HVHE))
309 return;
310
311 /*
312 * Leap of faith: at this point, we must be running VHE one way or
313 * another, and FEAT_PAN is required to be implemented. If KVM
314 * explodes at runtime because your system does not abide by this
315 * requirement, call your favourite HW vendor, they have screwed up.
316 *
317 * We don't expect hVHE to access any userspace mapping, so always
318 * set PSTATE.PAN on enty. Same thing if we have PAN enabled on an
319 * EL2 kernel. Only force it to 0 if we have not configured PAN in
320 * the kernel (and you know this is really silly).
321 */
322 if (cpus_have_cap(ARM64_KVM_HVHE) || IS_ENABLED(CONFIG_ARM64_PAN))
323 *updptr = cpu_to_le32(ENCODE_PSTATE(1, PAN));
324 else
325 *updptr = cpu_to_le32(ENCODE_PSTATE(0, PAN));
326}
327

source code of linux/arch/arm64/kvm/va_layout.c