1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2010 SUSE Linux Products GmbH. All rights reserved. |
4 | * Copyright 2010-2011 Freescale Semiconductor, Inc. |
5 | * |
6 | * Authors: |
7 | * Alexander Graf <agraf@suse.de> |
8 | */ |
9 | |
10 | #include <linux/kvm_host.h> |
11 | #include <linux/init.h> |
12 | #include <linux/export.h> |
13 | #include <linux/kmemleak.h> |
14 | #include <linux/kvm_para.h> |
15 | #include <linux/slab.h> |
16 | #include <linux/of.h> |
17 | #include <linux/pagemap.h> |
18 | |
19 | #include <asm/reg.h> |
20 | #include <asm/sections.h> |
21 | #include <asm/cacheflush.h> |
22 | #include <asm/disassemble.h> |
23 | #include <asm/ppc-opcode.h> |
24 | #include <asm/epapr_hcalls.h> |
25 | |
26 | #define KVM_MAGIC_PAGE (-4096L) |
27 | #define magic_var(x) KVM_MAGIC_PAGE + offsetof(struct kvm_vcpu_arch_shared, x) |
28 | |
29 | #define KVM_INST_LWZ 0x80000000 |
30 | #define KVM_INST_STW 0x90000000 |
31 | #define KVM_INST_LD 0xe8000000 |
32 | #define KVM_INST_STD 0xf8000000 |
33 | #define KVM_INST_NOP 0x60000000 |
34 | #define KVM_INST_B 0x48000000 |
35 | #define KVM_INST_B_MASK 0x03ffffff |
36 | #define KVM_INST_B_MAX 0x01ffffff |
37 | #define KVM_INST_LI 0x38000000 |
38 | |
39 | #define KVM_MASK_RT 0x03e00000 |
40 | #define KVM_RT_30 0x03c00000 |
41 | #define KVM_MASK_RB 0x0000f800 |
42 | #define KVM_INST_MFMSR 0x7c0000a6 |
43 | |
44 | #define SPR_FROM 0 |
45 | #define SPR_TO 0x100 |
46 | |
47 | #define KVM_INST_SPR(sprn, moveto) (0x7c0002a6 | \ |
48 | (((sprn) & 0x1f) << 16) | \ |
49 | (((sprn) & 0x3e0) << 6) | \ |
50 | (moveto)) |
51 | |
52 | #define KVM_INST_MFSPR(sprn) KVM_INST_SPR(sprn, SPR_FROM) |
53 | #define KVM_INST_MTSPR(sprn) KVM_INST_SPR(sprn, SPR_TO) |
54 | |
55 | #define KVM_INST_TLBSYNC 0x7c00046c |
56 | #define KVM_INST_MTMSRD_L0 0x7c000164 |
57 | #define KVM_INST_MTMSRD_L1 0x7c010164 |
58 | #define KVM_INST_MTMSR 0x7c000124 |
59 | |
60 | #define KVM_INST_WRTEE 0x7c000106 |
61 | #define KVM_INST_WRTEEI_0 0x7c000146 |
62 | #define KVM_INST_WRTEEI_1 0x7c008146 |
63 | |
64 | #define KVM_INST_MTSRIN 0x7c0001e4 |
65 | |
66 | static bool kvm_patching_worked = true; |
67 | extern char kvm_tmp[]; |
68 | extern char kvm_tmp_end[]; |
69 | static int kvm_tmp_index; |
70 | |
71 | static void __init kvm_patch_ins(u32 *inst, u32 new_inst) |
72 | { |
73 | *inst = new_inst; |
74 | flush_icache_range(start: (ulong)inst, end: (ulong)inst + 4); |
75 | } |
76 | |
77 | static void __init kvm_patch_ins_ll(u32 *inst, long addr, u32 rt) |
78 | { |
79 | #ifdef CONFIG_64BIT |
80 | kvm_patch_ins(inst, KVM_INST_LD | rt | (addr & 0x0000fffc)); |
81 | #else |
82 | kvm_patch_ins(inst, KVM_INST_LWZ | rt | (addr & 0x0000fffc)); |
83 | #endif |
84 | } |
85 | |
86 | static void __init kvm_patch_ins_ld(u32 *inst, long addr, u32 rt) |
87 | { |
88 | #ifdef CONFIG_64BIT |
89 | kvm_patch_ins(inst, KVM_INST_LD | rt | (addr & 0x0000fffc)); |
90 | #else |
91 | kvm_patch_ins(inst, KVM_INST_LWZ | rt | ((addr + 4) & 0x0000fffc)); |
92 | #endif |
93 | } |
94 | |
95 | static void __init kvm_patch_ins_lwz(u32 *inst, long addr, u32 rt) |
96 | { |
97 | kvm_patch_ins(inst, KVM_INST_LWZ | rt | (addr & 0x0000ffff)); |
98 | } |
99 | |
100 | static void __init kvm_patch_ins_std(u32 *inst, long addr, u32 rt) |
101 | { |
102 | #ifdef CONFIG_64BIT |
103 | kvm_patch_ins(inst, KVM_INST_STD | rt | (addr & 0x0000fffc)); |
104 | #else |
105 | kvm_patch_ins(inst, KVM_INST_STW | rt | ((addr + 4) & 0x0000fffc)); |
106 | #endif |
107 | } |
108 | |
109 | static void __init kvm_patch_ins_stw(u32 *inst, long addr, u32 rt) |
110 | { |
111 | kvm_patch_ins(inst, KVM_INST_STW | rt | (addr & 0x0000fffc)); |
112 | } |
113 | |
114 | static void __init kvm_patch_ins_nop(u32 *inst) |
115 | { |
116 | kvm_patch_ins(inst, KVM_INST_NOP); |
117 | } |
118 | |
119 | static void __init kvm_patch_ins_b(u32 *inst, int addr) |
120 | { |
121 | #if defined(CONFIG_RELOCATABLE) && defined(CONFIG_PPC_BOOK3S) |
122 | /* On relocatable kernels interrupts handlers and our code |
123 | can be in different regions, so we don't patch them */ |
124 | |
125 | if ((ulong)inst < (ulong)&__end_interrupts) |
126 | return; |
127 | #endif |
128 | |
129 | kvm_patch_ins(inst, KVM_INST_B | (addr & KVM_INST_B_MASK)); |
130 | } |
131 | |
132 | static u32 * __init kvm_alloc(int len) |
133 | { |
134 | u32 *p; |
135 | |
136 | if ((kvm_tmp_index + len) > (kvm_tmp_end - kvm_tmp)) { |
137 | printk(KERN_ERR "KVM: No more space (%d + %d)\n" , |
138 | kvm_tmp_index, len); |
139 | kvm_patching_worked = false; |
140 | return NULL; |
141 | } |
142 | |
143 | p = (void*)&kvm_tmp[kvm_tmp_index]; |
144 | kvm_tmp_index += len; |
145 | |
146 | return p; |
147 | } |
148 | |
149 | extern u32 kvm_emulate_mtmsrd_branch_offs; |
150 | extern u32 kvm_emulate_mtmsrd_reg_offs; |
151 | extern u32 kvm_emulate_mtmsrd_orig_ins_offs; |
152 | extern u32 kvm_emulate_mtmsrd_len; |
153 | extern u32 kvm_emulate_mtmsrd[]; |
154 | |
155 | static void __init kvm_patch_ins_mtmsrd(u32 *inst, u32 rt) |
156 | { |
157 | u32 *p; |
158 | int distance_start; |
159 | int distance_end; |
160 | ulong next_inst; |
161 | |
162 | p = kvm_alloc(len: kvm_emulate_mtmsrd_len * 4); |
163 | if (!p) |
164 | return; |
165 | |
166 | /* Find out where we are and put everything there */ |
167 | distance_start = (ulong)p - (ulong)inst; |
168 | next_inst = ((ulong)inst + 4); |
169 | distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsrd_branch_offs]; |
170 | |
171 | /* Make sure we only write valid b instructions */ |
172 | if (distance_start > KVM_INST_B_MAX) { |
173 | kvm_patching_worked = false; |
174 | return; |
175 | } |
176 | |
177 | /* Modify the chunk to fit the invocation */ |
178 | memcpy(p, kvm_emulate_mtmsrd, kvm_emulate_mtmsrd_len * 4); |
179 | p[kvm_emulate_mtmsrd_branch_offs] |= distance_end & KVM_INST_B_MASK; |
180 | switch (get_rt(rt)) { |
181 | case 30: |
182 | kvm_patch_ins_ll(&p[kvm_emulate_mtmsrd_reg_offs], |
183 | magic_var(scratch2), KVM_RT_30); |
184 | break; |
185 | case 31: |
186 | kvm_patch_ins_ll(&p[kvm_emulate_mtmsrd_reg_offs], |
187 | magic_var(scratch1), KVM_RT_30); |
188 | break; |
189 | default: |
190 | p[kvm_emulate_mtmsrd_reg_offs] |= rt; |
191 | break; |
192 | } |
193 | |
194 | p[kvm_emulate_mtmsrd_orig_ins_offs] = *inst; |
195 | flush_icache_range(start: (ulong)p, end: (ulong)p + kvm_emulate_mtmsrd_len * 4); |
196 | |
197 | /* Patch the invocation */ |
198 | kvm_patch_ins_b(inst, addr: distance_start); |
199 | } |
200 | |
201 | extern u32 kvm_emulate_mtmsr_branch_offs; |
202 | extern u32 kvm_emulate_mtmsr_reg1_offs; |
203 | extern u32 kvm_emulate_mtmsr_reg2_offs; |
204 | extern u32 kvm_emulate_mtmsr_orig_ins_offs; |
205 | extern u32 kvm_emulate_mtmsr_len; |
206 | extern u32 kvm_emulate_mtmsr[]; |
207 | |
208 | static void __init kvm_patch_ins_mtmsr(u32 *inst, u32 rt) |
209 | { |
210 | u32 *p; |
211 | int distance_start; |
212 | int distance_end; |
213 | ulong next_inst; |
214 | |
215 | p = kvm_alloc(len: kvm_emulate_mtmsr_len * 4); |
216 | if (!p) |
217 | return; |
218 | |
219 | /* Find out where we are and put everything there */ |
220 | distance_start = (ulong)p - (ulong)inst; |
221 | next_inst = ((ulong)inst + 4); |
222 | distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsr_branch_offs]; |
223 | |
224 | /* Make sure we only write valid b instructions */ |
225 | if (distance_start > KVM_INST_B_MAX) { |
226 | kvm_patching_worked = false; |
227 | return; |
228 | } |
229 | |
230 | /* Modify the chunk to fit the invocation */ |
231 | memcpy(p, kvm_emulate_mtmsr, kvm_emulate_mtmsr_len * 4); |
232 | p[kvm_emulate_mtmsr_branch_offs] |= distance_end & KVM_INST_B_MASK; |
233 | |
234 | /* Make clobbered registers work too */ |
235 | switch (get_rt(rt)) { |
236 | case 30: |
237 | kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg1_offs], |
238 | magic_var(scratch2), KVM_RT_30); |
239 | kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg2_offs], |
240 | magic_var(scratch2), KVM_RT_30); |
241 | break; |
242 | case 31: |
243 | kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg1_offs], |
244 | magic_var(scratch1), KVM_RT_30); |
245 | kvm_patch_ins_ll(&p[kvm_emulate_mtmsr_reg2_offs], |
246 | magic_var(scratch1), KVM_RT_30); |
247 | break; |
248 | default: |
249 | p[kvm_emulate_mtmsr_reg1_offs] |= rt; |
250 | p[kvm_emulate_mtmsr_reg2_offs] |= rt; |
251 | break; |
252 | } |
253 | |
254 | p[kvm_emulate_mtmsr_orig_ins_offs] = *inst; |
255 | flush_icache_range(start: (ulong)p, end: (ulong)p + kvm_emulate_mtmsr_len * 4); |
256 | |
257 | /* Patch the invocation */ |
258 | kvm_patch_ins_b(inst, addr: distance_start); |
259 | } |
260 | |
261 | #ifdef CONFIG_BOOKE |
262 | |
263 | extern u32 kvm_emulate_wrtee_branch_offs; |
264 | extern u32 kvm_emulate_wrtee_reg_offs; |
265 | extern u32 kvm_emulate_wrtee_orig_ins_offs; |
266 | extern u32 kvm_emulate_wrtee_len; |
267 | extern u32 kvm_emulate_wrtee[]; |
268 | |
269 | static void __init kvm_patch_ins_wrtee(u32 *inst, u32 rt, int imm_one) |
270 | { |
271 | u32 *p; |
272 | int distance_start; |
273 | int distance_end; |
274 | ulong next_inst; |
275 | |
276 | p = kvm_alloc(kvm_emulate_wrtee_len * 4); |
277 | if (!p) |
278 | return; |
279 | |
280 | /* Find out where we are and put everything there */ |
281 | distance_start = (ulong)p - (ulong)inst; |
282 | next_inst = ((ulong)inst + 4); |
283 | distance_end = next_inst - (ulong)&p[kvm_emulate_wrtee_branch_offs]; |
284 | |
285 | /* Make sure we only write valid b instructions */ |
286 | if (distance_start > KVM_INST_B_MAX) { |
287 | kvm_patching_worked = false; |
288 | return; |
289 | } |
290 | |
291 | /* Modify the chunk to fit the invocation */ |
292 | memcpy(p, kvm_emulate_wrtee, kvm_emulate_wrtee_len * 4); |
293 | p[kvm_emulate_wrtee_branch_offs] |= distance_end & KVM_INST_B_MASK; |
294 | |
295 | if (imm_one) { |
296 | p[kvm_emulate_wrtee_reg_offs] = |
297 | KVM_INST_LI | __PPC_RT(R30) | MSR_EE; |
298 | } else { |
299 | /* Make clobbered registers work too */ |
300 | switch (get_rt(rt)) { |
301 | case 30: |
302 | kvm_patch_ins_ll(&p[kvm_emulate_wrtee_reg_offs], |
303 | magic_var(scratch2), KVM_RT_30); |
304 | break; |
305 | case 31: |
306 | kvm_patch_ins_ll(&p[kvm_emulate_wrtee_reg_offs], |
307 | magic_var(scratch1), KVM_RT_30); |
308 | break; |
309 | default: |
310 | p[kvm_emulate_wrtee_reg_offs] |= rt; |
311 | break; |
312 | } |
313 | } |
314 | |
315 | p[kvm_emulate_wrtee_orig_ins_offs] = *inst; |
316 | flush_icache_range((ulong)p, (ulong)p + kvm_emulate_wrtee_len * 4); |
317 | |
318 | /* Patch the invocation */ |
319 | kvm_patch_ins_b(inst, distance_start); |
320 | } |
321 | |
322 | extern u32 kvm_emulate_wrteei_0_branch_offs; |
323 | extern u32 kvm_emulate_wrteei_0_len; |
324 | extern u32 kvm_emulate_wrteei_0[]; |
325 | |
326 | static void __init kvm_patch_ins_wrteei_0(u32 *inst) |
327 | { |
328 | u32 *p; |
329 | int distance_start; |
330 | int distance_end; |
331 | ulong next_inst; |
332 | |
333 | p = kvm_alloc(kvm_emulate_wrteei_0_len * 4); |
334 | if (!p) |
335 | return; |
336 | |
337 | /* Find out where we are and put everything there */ |
338 | distance_start = (ulong)p - (ulong)inst; |
339 | next_inst = ((ulong)inst + 4); |
340 | distance_end = next_inst - (ulong)&p[kvm_emulate_wrteei_0_branch_offs]; |
341 | |
342 | /* Make sure we only write valid b instructions */ |
343 | if (distance_start > KVM_INST_B_MAX) { |
344 | kvm_patching_worked = false; |
345 | return; |
346 | } |
347 | |
348 | memcpy(p, kvm_emulate_wrteei_0, kvm_emulate_wrteei_0_len * 4); |
349 | p[kvm_emulate_wrteei_0_branch_offs] |= distance_end & KVM_INST_B_MASK; |
350 | flush_icache_range((ulong)p, (ulong)p + kvm_emulate_wrteei_0_len * 4); |
351 | |
352 | /* Patch the invocation */ |
353 | kvm_patch_ins_b(inst, distance_start); |
354 | } |
355 | |
356 | #endif |
357 | |
358 | #ifdef CONFIG_PPC_BOOK3S_32 |
359 | |
360 | extern u32 kvm_emulate_mtsrin_branch_offs; |
361 | extern u32 kvm_emulate_mtsrin_reg1_offs; |
362 | extern u32 kvm_emulate_mtsrin_reg2_offs; |
363 | extern u32 kvm_emulate_mtsrin_orig_ins_offs; |
364 | extern u32 kvm_emulate_mtsrin_len; |
365 | extern u32 kvm_emulate_mtsrin[]; |
366 | |
367 | static void __init kvm_patch_ins_mtsrin(u32 *inst, u32 rt, u32 rb) |
368 | { |
369 | u32 *p; |
370 | int distance_start; |
371 | int distance_end; |
372 | ulong next_inst; |
373 | |
374 | p = kvm_alloc(kvm_emulate_mtsrin_len * 4); |
375 | if (!p) |
376 | return; |
377 | |
378 | /* Find out where we are and put everything there */ |
379 | distance_start = (ulong)p - (ulong)inst; |
380 | next_inst = ((ulong)inst + 4); |
381 | distance_end = next_inst - (ulong)&p[kvm_emulate_mtsrin_branch_offs]; |
382 | |
383 | /* Make sure we only write valid b instructions */ |
384 | if (distance_start > KVM_INST_B_MAX) { |
385 | kvm_patching_worked = false; |
386 | return; |
387 | } |
388 | |
389 | /* Modify the chunk to fit the invocation */ |
390 | memcpy(p, kvm_emulate_mtsrin, kvm_emulate_mtsrin_len * 4); |
391 | p[kvm_emulate_mtsrin_branch_offs] |= distance_end & KVM_INST_B_MASK; |
392 | p[kvm_emulate_mtsrin_reg1_offs] |= (rb << 10); |
393 | p[kvm_emulate_mtsrin_reg2_offs] |= rt; |
394 | p[kvm_emulate_mtsrin_orig_ins_offs] = *inst; |
395 | flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtsrin_len * 4); |
396 | |
397 | /* Patch the invocation */ |
398 | kvm_patch_ins_b(inst, distance_start); |
399 | } |
400 | |
401 | #endif |
402 | |
403 | static void __init kvm_map_magic_page(void *data) |
404 | { |
405 | u32 *features = data; |
406 | |
407 | ulong in[8] = {0}; |
408 | ulong out[8]; |
409 | |
410 | in[0] = KVM_MAGIC_PAGE; |
411 | in[1] = KVM_MAGIC_PAGE | MAGIC_PAGE_FLAG_NOT_MAPPED_NX; |
412 | |
413 | epapr_hypercall(in, out, KVM_HCALL_TOKEN(KVM_HC_PPC_MAP_MAGIC_PAGE)); |
414 | |
415 | *features = out[0]; |
416 | } |
417 | |
418 | static void __init kvm_check_ins(u32 *inst, u32 features) |
419 | { |
420 | u32 _inst = *inst; |
421 | u32 inst_no_rt = _inst & ~KVM_MASK_RT; |
422 | u32 inst_rt = _inst & KVM_MASK_RT; |
423 | |
424 | switch (inst_no_rt) { |
425 | /* Loads */ |
426 | case KVM_INST_MFMSR: |
427 | kvm_patch_ins_ld(inst, magic_var(msr), inst_rt); |
428 | break; |
429 | case KVM_INST_MFSPR(SPRN_SPRG0): |
430 | kvm_patch_ins_ld(inst, magic_var(sprg0), inst_rt); |
431 | break; |
432 | case KVM_INST_MFSPR(SPRN_SPRG1): |
433 | kvm_patch_ins_ld(inst, magic_var(sprg1), inst_rt); |
434 | break; |
435 | case KVM_INST_MFSPR(SPRN_SPRG2): |
436 | kvm_patch_ins_ld(inst, magic_var(sprg2), inst_rt); |
437 | break; |
438 | case KVM_INST_MFSPR(SPRN_SPRG3): |
439 | kvm_patch_ins_ld(inst, magic_var(sprg3), inst_rt); |
440 | break; |
441 | case KVM_INST_MFSPR(SPRN_SRR0): |
442 | kvm_patch_ins_ld(inst, magic_var(srr0), inst_rt); |
443 | break; |
444 | case KVM_INST_MFSPR(SPRN_SRR1): |
445 | kvm_patch_ins_ld(inst, magic_var(srr1), inst_rt); |
446 | break; |
447 | #ifdef CONFIG_BOOKE |
448 | case KVM_INST_MFSPR(SPRN_DEAR): |
449 | #else |
450 | case KVM_INST_MFSPR(SPRN_DAR): |
451 | #endif |
452 | kvm_patch_ins_ld(inst, magic_var(dar), inst_rt); |
453 | break; |
454 | case KVM_INST_MFSPR(SPRN_DSISR): |
455 | kvm_patch_ins_lwz(inst, magic_var(dsisr), inst_rt); |
456 | break; |
457 | |
458 | #ifdef CONFIG_PPC_E500 |
459 | case KVM_INST_MFSPR(SPRN_MAS0): |
460 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
461 | kvm_patch_ins_lwz(inst, magic_var(mas0), inst_rt); |
462 | break; |
463 | case KVM_INST_MFSPR(SPRN_MAS1): |
464 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
465 | kvm_patch_ins_lwz(inst, magic_var(mas1), inst_rt); |
466 | break; |
467 | case KVM_INST_MFSPR(SPRN_MAS2): |
468 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
469 | kvm_patch_ins_ld(inst, magic_var(mas2), inst_rt); |
470 | break; |
471 | case KVM_INST_MFSPR(SPRN_MAS3): |
472 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
473 | kvm_patch_ins_lwz(inst, magic_var(mas7_3) + 4, inst_rt); |
474 | break; |
475 | case KVM_INST_MFSPR(SPRN_MAS4): |
476 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
477 | kvm_patch_ins_lwz(inst, magic_var(mas4), inst_rt); |
478 | break; |
479 | case KVM_INST_MFSPR(SPRN_MAS6): |
480 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
481 | kvm_patch_ins_lwz(inst, magic_var(mas6), inst_rt); |
482 | break; |
483 | case KVM_INST_MFSPR(SPRN_MAS7): |
484 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
485 | kvm_patch_ins_lwz(inst, magic_var(mas7_3), inst_rt); |
486 | break; |
487 | #endif /* CONFIG_PPC_E500 */ |
488 | |
489 | case KVM_INST_MFSPR(SPRN_SPRG4): |
490 | #ifdef CONFIG_BOOKE |
491 | case KVM_INST_MFSPR(SPRN_SPRG4R): |
492 | #endif |
493 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
494 | kvm_patch_ins_ld(inst, magic_var(sprg4), inst_rt); |
495 | break; |
496 | case KVM_INST_MFSPR(SPRN_SPRG5): |
497 | #ifdef CONFIG_BOOKE |
498 | case KVM_INST_MFSPR(SPRN_SPRG5R): |
499 | #endif |
500 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
501 | kvm_patch_ins_ld(inst, magic_var(sprg5), inst_rt); |
502 | break; |
503 | case KVM_INST_MFSPR(SPRN_SPRG6): |
504 | #ifdef CONFIG_BOOKE |
505 | case KVM_INST_MFSPR(SPRN_SPRG6R): |
506 | #endif |
507 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
508 | kvm_patch_ins_ld(inst, magic_var(sprg6), inst_rt); |
509 | break; |
510 | case KVM_INST_MFSPR(SPRN_SPRG7): |
511 | #ifdef CONFIG_BOOKE |
512 | case KVM_INST_MFSPR(SPRN_SPRG7R): |
513 | #endif |
514 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
515 | kvm_patch_ins_ld(inst, magic_var(sprg7), inst_rt); |
516 | break; |
517 | |
518 | #ifdef CONFIG_BOOKE |
519 | case KVM_INST_MFSPR(SPRN_ESR): |
520 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
521 | kvm_patch_ins_lwz(inst, magic_var(esr), inst_rt); |
522 | break; |
523 | #endif |
524 | |
525 | case KVM_INST_MFSPR(SPRN_PIR): |
526 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
527 | kvm_patch_ins_lwz(inst, magic_var(pir), inst_rt); |
528 | break; |
529 | |
530 | |
531 | /* Stores */ |
532 | case KVM_INST_MTSPR(SPRN_SPRG0): |
533 | kvm_patch_ins_std(inst, magic_var(sprg0), inst_rt); |
534 | break; |
535 | case KVM_INST_MTSPR(SPRN_SPRG1): |
536 | kvm_patch_ins_std(inst, magic_var(sprg1), inst_rt); |
537 | break; |
538 | case KVM_INST_MTSPR(SPRN_SPRG2): |
539 | kvm_patch_ins_std(inst, magic_var(sprg2), inst_rt); |
540 | break; |
541 | case KVM_INST_MTSPR(SPRN_SPRG3): |
542 | kvm_patch_ins_std(inst, magic_var(sprg3), inst_rt); |
543 | break; |
544 | case KVM_INST_MTSPR(SPRN_SRR0): |
545 | kvm_patch_ins_std(inst, magic_var(srr0), inst_rt); |
546 | break; |
547 | case KVM_INST_MTSPR(SPRN_SRR1): |
548 | kvm_patch_ins_std(inst, magic_var(srr1), inst_rt); |
549 | break; |
550 | #ifdef CONFIG_BOOKE |
551 | case KVM_INST_MTSPR(SPRN_DEAR): |
552 | #else |
553 | case KVM_INST_MTSPR(SPRN_DAR): |
554 | #endif |
555 | kvm_patch_ins_std(inst, magic_var(dar), inst_rt); |
556 | break; |
557 | case KVM_INST_MTSPR(SPRN_DSISR): |
558 | kvm_patch_ins_stw(inst, magic_var(dsisr), inst_rt); |
559 | break; |
560 | #ifdef CONFIG_PPC_E500 |
561 | case KVM_INST_MTSPR(SPRN_MAS0): |
562 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
563 | kvm_patch_ins_stw(inst, magic_var(mas0), inst_rt); |
564 | break; |
565 | case KVM_INST_MTSPR(SPRN_MAS1): |
566 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
567 | kvm_patch_ins_stw(inst, magic_var(mas1), inst_rt); |
568 | break; |
569 | case KVM_INST_MTSPR(SPRN_MAS2): |
570 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
571 | kvm_patch_ins_std(inst, magic_var(mas2), inst_rt); |
572 | break; |
573 | case KVM_INST_MTSPR(SPRN_MAS3): |
574 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
575 | kvm_patch_ins_stw(inst, magic_var(mas7_3) + 4, inst_rt); |
576 | break; |
577 | case KVM_INST_MTSPR(SPRN_MAS4): |
578 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
579 | kvm_patch_ins_stw(inst, magic_var(mas4), inst_rt); |
580 | break; |
581 | case KVM_INST_MTSPR(SPRN_MAS6): |
582 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
583 | kvm_patch_ins_stw(inst, magic_var(mas6), inst_rt); |
584 | break; |
585 | case KVM_INST_MTSPR(SPRN_MAS7): |
586 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
587 | kvm_patch_ins_stw(inst, magic_var(mas7_3), inst_rt); |
588 | break; |
589 | #endif /* CONFIG_PPC_E500 */ |
590 | |
591 | case KVM_INST_MTSPR(SPRN_SPRG4): |
592 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
593 | kvm_patch_ins_std(inst, magic_var(sprg4), inst_rt); |
594 | break; |
595 | case KVM_INST_MTSPR(SPRN_SPRG5): |
596 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
597 | kvm_patch_ins_std(inst, magic_var(sprg5), inst_rt); |
598 | break; |
599 | case KVM_INST_MTSPR(SPRN_SPRG6): |
600 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
601 | kvm_patch_ins_std(inst, magic_var(sprg6), inst_rt); |
602 | break; |
603 | case KVM_INST_MTSPR(SPRN_SPRG7): |
604 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
605 | kvm_patch_ins_std(inst, magic_var(sprg7), inst_rt); |
606 | break; |
607 | |
608 | #ifdef CONFIG_BOOKE |
609 | case KVM_INST_MTSPR(SPRN_ESR): |
610 | if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) |
611 | kvm_patch_ins_stw(inst, magic_var(esr), inst_rt); |
612 | break; |
613 | #endif |
614 | |
615 | /* Nops */ |
616 | case KVM_INST_TLBSYNC: |
617 | kvm_patch_ins_nop(inst); |
618 | break; |
619 | |
620 | /* Rewrites */ |
621 | case KVM_INST_MTMSRD_L1: |
622 | kvm_patch_ins_mtmsrd(inst, rt: inst_rt); |
623 | break; |
624 | case KVM_INST_MTMSR: |
625 | case KVM_INST_MTMSRD_L0: |
626 | kvm_patch_ins_mtmsr(inst, rt: inst_rt); |
627 | break; |
628 | #ifdef CONFIG_BOOKE |
629 | case KVM_INST_WRTEE: |
630 | kvm_patch_ins_wrtee(inst, inst_rt, 0); |
631 | break; |
632 | #endif |
633 | } |
634 | |
635 | switch (inst_no_rt & ~KVM_MASK_RB) { |
636 | #ifdef CONFIG_PPC_BOOK3S_32 |
637 | case KVM_INST_MTSRIN: |
638 | if (features & KVM_MAGIC_FEAT_SR) { |
639 | u32 inst_rb = _inst & KVM_MASK_RB; |
640 | kvm_patch_ins_mtsrin(inst, inst_rt, inst_rb); |
641 | } |
642 | break; |
643 | #endif |
644 | } |
645 | |
646 | switch (_inst) { |
647 | #ifdef CONFIG_BOOKE |
648 | case KVM_INST_WRTEEI_0: |
649 | kvm_patch_ins_wrteei_0(inst); |
650 | break; |
651 | |
652 | case KVM_INST_WRTEEI_1: |
653 | kvm_patch_ins_wrtee(inst, 0, 1); |
654 | break; |
655 | #endif |
656 | } |
657 | } |
658 | |
659 | extern u32 kvm_template_start[]; |
660 | extern u32 kvm_template_end[]; |
661 | |
662 | static void __init kvm_use_magic_page(void) |
663 | { |
664 | u32 *p; |
665 | u32 *start, *end; |
666 | u32 features; |
667 | |
668 | /* Tell the host to map the magic page to -4096 on all CPUs */ |
669 | on_each_cpu(func: kvm_map_magic_page, info: &features, wait: 1); |
670 | |
671 | /* Quick self-test to see if the mapping works */ |
672 | if (fault_in_readable(uaddr: (const char __user *)KVM_MAGIC_PAGE, |
673 | size: sizeof(u32))) { |
674 | kvm_patching_worked = false; |
675 | return; |
676 | } |
677 | |
678 | /* Now loop through all code and find instructions */ |
679 | start = (void*)_stext; |
680 | end = (void*)_etext; |
681 | |
682 | /* |
683 | * Being interrupted in the middle of patching would |
684 | * be bad for SPRG4-7, which KVM can't keep in sync |
685 | * with emulated accesses because reads don't trap. |
686 | */ |
687 | local_irq_disable(); |
688 | |
689 | for (p = start; p < end; p++) { |
690 | /* Avoid patching the template code */ |
691 | if (p >= kvm_template_start && p < kvm_template_end) { |
692 | p = kvm_template_end - 1; |
693 | continue; |
694 | } |
695 | kvm_check_ins(inst: p, features); |
696 | } |
697 | |
698 | local_irq_enable(); |
699 | |
700 | printk(KERN_INFO "KVM: Live patching for a fast VM %s\n" , |
701 | kvm_patching_worked ? "worked" : "failed" ); |
702 | } |
703 | |
704 | static int __init kvm_guest_init(void) |
705 | { |
706 | if (!kvm_para_available()) |
707 | return 0; |
708 | |
709 | if (!epapr_paravirt_enabled) |
710 | return 0; |
711 | |
712 | if (kvm_para_has_feature(KVM_FEATURE_MAGIC_PAGE)) |
713 | kvm_use_magic_page(); |
714 | |
715 | #ifdef CONFIG_PPC_BOOK3S_64 |
716 | /* Enable napping */ |
717 | powersave_nap = 1; |
718 | #endif |
719 | |
720 | return 0; |
721 | } |
722 | |
723 | postcore_initcall(kvm_guest_init); |
724 | |