1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2019 Western Digital Corporation or its affiliates. |
4 | * |
5 | * Authors: |
6 | * Anup Patel <anup.patel@wdc.com> |
7 | */ |
8 | |
9 | #include <linux/bitops.h> |
10 | #include <linux/entry-kvm.h> |
11 | #include <linux/errno.h> |
12 | #include <linux/err.h> |
13 | #include <linux/kdebug.h> |
14 | #include <linux/module.h> |
15 | #include <linux/percpu.h> |
16 | #include <linux/vmalloc.h> |
17 | #include <linux/sched/signal.h> |
18 | #include <linux/fs.h> |
19 | #include <linux/kvm_host.h> |
20 | #include <asm/csr.h> |
21 | #include <asm/cacheflush.h> |
22 | #include <asm/kvm_vcpu_vector.h> |
23 | |
24 | const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { |
25 | KVM_GENERIC_VCPU_STATS(), |
26 | STATS_DESC_COUNTER(VCPU, ecall_exit_stat), |
27 | STATS_DESC_COUNTER(VCPU, wfi_exit_stat), |
28 | STATS_DESC_COUNTER(VCPU, mmio_exit_user), |
29 | STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), |
30 | STATS_DESC_COUNTER(VCPU, csr_exit_user), |
31 | STATS_DESC_COUNTER(VCPU, csr_exit_kernel), |
32 | STATS_DESC_COUNTER(VCPU, signal_exits), |
33 | STATS_DESC_COUNTER(VCPU, exits) |
34 | }; |
35 | |
36 | const struct kvm_stats_header = { |
37 | .name_size = KVM_STATS_NAME_SIZE, |
38 | .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), |
39 | .id_offset = sizeof(struct kvm_stats_header), |
40 | .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, |
41 | .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + |
42 | sizeof(kvm_vcpu_stats_desc), |
43 | }; |
44 | |
45 | static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) |
46 | { |
47 | struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; |
48 | struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; |
49 | struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; |
50 | struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context; |
51 | bool loaded; |
52 | |
53 | /** |
54 | * The preemption should be disabled here because it races with |
55 | * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which |
56 | * also calls vcpu_load/put. |
57 | */ |
58 | get_cpu(); |
59 | loaded = (vcpu->cpu != -1); |
60 | if (loaded) |
61 | kvm_arch_vcpu_put(vcpu); |
62 | |
63 | vcpu->arch.last_exit_cpu = -1; |
64 | |
65 | memcpy(csr, reset_csr, sizeof(*csr)); |
66 | |
67 | memcpy(cntx, reset_cntx, sizeof(*cntx)); |
68 | |
69 | kvm_riscv_vcpu_fp_reset(vcpu); |
70 | |
71 | kvm_riscv_vcpu_vector_reset(vcpu); |
72 | |
73 | kvm_riscv_vcpu_timer_reset(vcpu); |
74 | |
75 | kvm_riscv_vcpu_aia_reset(vcpu); |
76 | |
77 | bitmap_zero(vcpu->arch.dst: irqs_pending, nbits: KVM_RISCV_VCPU_NR_IRQS); |
78 | bitmap_zero(dst: vcpu->arch.irqs_pending_mask, nbits: KVM_RISCV_VCPU_NR_IRQS); |
79 | |
80 | kvm_riscv_vcpu_pmu_reset(vcpu); |
81 | |
82 | vcpu->arch.hfence_head = 0; |
83 | vcpu->arch.hfence_tail = 0; |
84 | memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue)); |
85 | |
86 | kvm_riscv_vcpu_sbi_sta_reset(vcpu); |
87 | |
88 | /* Reset the guest CSRs for hotplug usecase */ |
89 | if (loaded) |
90 | kvm_arch_vcpu_load(vcpu, smp_processor_id()); |
91 | put_cpu(); |
92 | } |
93 | |
94 | int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) |
95 | { |
96 | return 0; |
97 | } |
98 | |
99 | int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) |
100 | { |
101 | int rc; |
102 | struct kvm_cpu_context *cntx; |
103 | struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; |
104 | |
105 | /* Mark this VCPU never ran */ |
106 | vcpu->arch.ran_atleast_once = false; |
107 | vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; |
108 | bitmap_zero(dst: vcpu->arch.isa, nbits: RISCV_ISA_EXT_MAX); |
109 | |
110 | /* Setup ISA features available to VCPU */ |
111 | kvm_riscv_vcpu_setup_isa(vcpu); |
112 | |
113 | /* Setup vendor, arch, and implementation details */ |
114 | vcpu->arch.mvendorid = sbi_get_mvendorid(); |
115 | vcpu->arch.marchid = sbi_get_marchid(); |
116 | vcpu->arch.mimpid = sbi_get_mimpid(); |
117 | |
118 | /* Setup VCPU hfence queue */ |
119 | spin_lock_init(&vcpu->arch.hfence_lock); |
120 | |
121 | /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */ |
122 | cntx = &vcpu->arch.guest_reset_context; |
123 | cntx->sstatus = SR_SPP | SR_SPIE; |
124 | cntx->hstatus = 0; |
125 | cntx->hstatus |= HSTATUS_VTW; |
126 | cntx->hstatus |= HSTATUS_SPVP; |
127 | cntx->hstatus |= HSTATUS_SPV; |
128 | |
129 | if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx)) |
130 | return -ENOMEM; |
131 | |
132 | /* By default, make CY, TM, and IR counters accessible in VU mode */ |
133 | reset_csr->scounteren = 0x7; |
134 | |
135 | /* Setup VCPU timer */ |
136 | kvm_riscv_vcpu_timer_init(vcpu); |
137 | |
138 | /* setup performance monitoring */ |
139 | kvm_riscv_vcpu_pmu_init(vcpu); |
140 | |
141 | /* Setup VCPU AIA */ |
142 | rc = kvm_riscv_vcpu_aia_init(vcpu); |
143 | if (rc) |
144 | return rc; |
145 | |
146 | /* |
147 | * Setup SBI extensions |
148 | * NOTE: This must be the last thing to be initialized. |
149 | */ |
150 | kvm_riscv_vcpu_sbi_init(vcpu); |
151 | |
152 | /* Reset VCPU */ |
153 | kvm_riscv_reset_vcpu(vcpu); |
154 | |
155 | return 0; |
156 | } |
157 | |
158 | void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) |
159 | { |
160 | /** |
161 | * vcpu with id 0 is the designated boot cpu. |
162 | * Keep all vcpus with non-zero id in power-off state so that |
163 | * they can be brought up using SBI HSM extension. |
164 | */ |
165 | if (vcpu->vcpu_idx != 0) |
166 | kvm_riscv_vcpu_power_off(vcpu); |
167 | } |
168 | |
169 | void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) |
170 | { |
171 | /* Cleanup VCPU AIA context */ |
172 | kvm_riscv_vcpu_aia_deinit(vcpu); |
173 | |
174 | /* Cleanup VCPU timer */ |
175 | kvm_riscv_vcpu_timer_deinit(vcpu); |
176 | |
177 | kvm_riscv_vcpu_pmu_deinit(vcpu); |
178 | |
179 | /* Free unused pages pre-allocated for G-stage page table mappings */ |
180 | kvm_mmu_free_memory_cache(mc: &vcpu->arch.mmu_page_cache); |
181 | |
182 | /* Free vector context space for host and guest kernel */ |
183 | kvm_riscv_vcpu_free_vector_context(vcpu); |
184 | } |
185 | |
186 | int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) |
187 | { |
188 | return kvm_riscv_vcpu_timer_pending(vcpu); |
189 | } |
190 | |
191 | void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) |
192 | { |
193 | kvm_riscv_aia_wakeon_hgei(vcpu, true); |
194 | } |
195 | |
196 | void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) |
197 | { |
198 | kvm_riscv_aia_wakeon_hgei(vcpu, false); |
199 | } |
200 | |
201 | int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) |
202 | { |
203 | return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) && |
204 | !vcpu->arch.power_off && !vcpu->arch.pause); |
205 | } |
206 | |
207 | int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) |
208 | { |
209 | return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; |
210 | } |
211 | |
212 | bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) |
213 | { |
214 | return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false; |
215 | } |
216 | |
217 | vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) |
218 | { |
219 | return VM_FAULT_SIGBUS; |
220 | } |
221 | |
222 | long kvm_arch_vcpu_async_ioctl(struct file *filp, |
223 | unsigned int ioctl, unsigned long arg) |
224 | { |
225 | struct kvm_vcpu *vcpu = filp->private_data; |
226 | void __user *argp = (void __user *)arg; |
227 | |
228 | if (ioctl == KVM_INTERRUPT) { |
229 | struct kvm_interrupt irq; |
230 | |
231 | if (copy_from_user(to: &irq, from: argp, n: sizeof(irq))) |
232 | return -EFAULT; |
233 | |
234 | if (irq.irq == KVM_INTERRUPT_SET) |
235 | return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT); |
236 | else |
237 | return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); |
238 | } |
239 | |
240 | return -ENOIOCTLCMD; |
241 | } |
242 | |
243 | long kvm_arch_vcpu_ioctl(struct file *filp, |
244 | unsigned int ioctl, unsigned long arg) |
245 | { |
246 | struct kvm_vcpu *vcpu = filp->private_data; |
247 | void __user *argp = (void __user *)arg; |
248 | long r = -EINVAL; |
249 | |
250 | switch (ioctl) { |
251 | case KVM_SET_ONE_REG: |
252 | case KVM_GET_ONE_REG: { |
253 | struct kvm_one_reg reg; |
254 | |
255 | r = -EFAULT; |
256 | if (copy_from_user(to: ®, from: argp, n: sizeof(reg))) |
257 | break; |
258 | |
259 | if (ioctl == KVM_SET_ONE_REG) |
260 | r = kvm_riscv_vcpu_set_reg(vcpu, ®); |
261 | else |
262 | r = kvm_riscv_vcpu_get_reg(vcpu, ®); |
263 | break; |
264 | } |
265 | case KVM_GET_REG_LIST: { |
266 | struct kvm_reg_list __user *user_list = argp; |
267 | struct kvm_reg_list reg_list; |
268 | unsigned int n; |
269 | |
270 | r = -EFAULT; |
271 | if (copy_from_user(to: ®_list, from: user_list, n: sizeof(reg_list))) |
272 | break; |
273 | n = reg_list.n; |
274 | reg_list.n = kvm_riscv_vcpu_num_regs(vcpu); |
275 | if (copy_to_user(to: user_list, from: ®_list, n: sizeof(reg_list))) |
276 | break; |
277 | r = -E2BIG; |
278 | if (n < reg_list.n) |
279 | break; |
280 | r = kvm_riscv_vcpu_copy_reg_indices(vcpu, user_list->reg); |
281 | break; |
282 | } |
283 | default: |
284 | break; |
285 | } |
286 | |
287 | return r; |
288 | } |
289 | |
290 | int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, |
291 | struct kvm_sregs *sregs) |
292 | { |
293 | return -EINVAL; |
294 | } |
295 | |
296 | int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, |
297 | struct kvm_sregs *sregs) |
298 | { |
299 | return -EINVAL; |
300 | } |
301 | |
302 | int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) |
303 | { |
304 | return -EINVAL; |
305 | } |
306 | |
307 | int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) |
308 | { |
309 | return -EINVAL; |
310 | } |
311 | |
312 | int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, |
313 | struct kvm_translation *tr) |
314 | { |
315 | return -EINVAL; |
316 | } |
317 | |
318 | int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) |
319 | { |
320 | return -EINVAL; |
321 | } |
322 | |
323 | int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) |
324 | { |
325 | return -EINVAL; |
326 | } |
327 | |
328 | void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) |
329 | { |
330 | struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; |
331 | unsigned long mask, val; |
332 | |
333 | if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { |
334 | mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); |
335 | val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; |
336 | |
337 | csr->hvip &= ~mask; |
338 | csr->hvip |= val; |
339 | } |
340 | |
341 | /* Flush AIA high interrupts */ |
342 | kvm_riscv_vcpu_aia_flush_interrupts(vcpu); |
343 | } |
344 | |
345 | void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) |
346 | { |
347 | unsigned long hvip; |
348 | struct kvm_vcpu_arch *v = &vcpu->arch; |
349 | struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; |
350 | |
351 | /* Read current HVIP and VSIE CSRs */ |
352 | csr->vsie = csr_read(CSR_VSIE); |
353 | |
354 | /* Sync-up HVIP.VSSIP bit changes does by Guest */ |
355 | hvip = csr_read(CSR_HVIP); |
356 | if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { |
357 | if (hvip & (1UL << IRQ_VS_SOFT)) { |
358 | if (!test_and_set_bit(IRQ_VS_SOFT, |
359 | v->irqs_pending_mask)) |
360 | set_bit(IRQ_VS_SOFT, v->irqs_pending); |
361 | } else { |
362 | if (!test_and_set_bit(IRQ_VS_SOFT, |
363 | v->irqs_pending_mask)) |
364 | clear_bit(IRQ_VS_SOFT, v->irqs_pending); |
365 | } |
366 | } |
367 | |
368 | /* Sync-up AIA high interrupts */ |
369 | kvm_riscv_vcpu_aia_sync_interrupts(vcpu); |
370 | |
371 | /* Sync-up timer CSRs */ |
372 | kvm_riscv_vcpu_timer_sync(vcpu); |
373 | } |
374 | |
375 | int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) |
376 | { |
377 | /* |
378 | * We only allow VS-mode software, timer, and external |
379 | * interrupts when irq is one of the local interrupts |
380 | * defined by RISC-V privilege specification. |
381 | */ |
382 | if (irq < IRQ_LOCAL_MAX && |
383 | irq != IRQ_VS_SOFT && |
384 | irq != IRQ_VS_TIMER && |
385 | irq != IRQ_VS_EXT) |
386 | return -EINVAL; |
387 | |
388 | set_bit(nr: irq, addr: vcpu->arch.irqs_pending); |
389 | smp_mb__before_atomic(); |
390 | set_bit(nr: irq, addr: vcpu->arch.irqs_pending_mask); |
391 | |
392 | kvm_vcpu_kick(vcpu); |
393 | |
394 | return 0; |
395 | } |
396 | |
397 | int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) |
398 | { |
399 | /* |
400 | * We only allow VS-mode software, timer, and external |
401 | * interrupts when irq is one of the local interrupts |
402 | * defined by RISC-V privilege specification. |
403 | */ |
404 | if (irq < IRQ_LOCAL_MAX && |
405 | irq != IRQ_VS_SOFT && |
406 | irq != IRQ_VS_TIMER && |
407 | irq != IRQ_VS_EXT) |
408 | return -EINVAL; |
409 | |
410 | clear_bit(nr: irq, addr: vcpu->arch.irqs_pending); |
411 | smp_mb__before_atomic(); |
412 | set_bit(nr: irq, addr: vcpu->arch.irqs_pending_mask); |
413 | |
414 | return 0; |
415 | } |
416 | |
417 | bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) |
418 | { |
419 | unsigned long ie; |
420 | |
421 | ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) |
422 | << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; |
423 | ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & |
424 | (unsigned long)mask; |
425 | if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) |
426 | return true; |
427 | |
428 | /* Check AIA high interrupts */ |
429 | return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); |
430 | } |
431 | |
432 | void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) |
433 | { |
434 | vcpu->arch.power_off = true; |
435 | kvm_make_request(KVM_REQ_SLEEP, vcpu); |
436 | kvm_vcpu_kick(vcpu); |
437 | } |
438 | |
439 | void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu) |
440 | { |
441 | vcpu->arch.power_off = false; |
442 | kvm_vcpu_wake_up(vcpu); |
443 | } |
444 | |
445 | int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, |
446 | struct kvm_mp_state *mp_state) |
447 | { |
448 | if (vcpu->arch.power_off) |
449 | mp_state->mp_state = KVM_MP_STATE_STOPPED; |
450 | else |
451 | mp_state->mp_state = KVM_MP_STATE_RUNNABLE; |
452 | |
453 | return 0; |
454 | } |
455 | |
456 | int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, |
457 | struct kvm_mp_state *mp_state) |
458 | { |
459 | int ret = 0; |
460 | |
461 | switch (mp_state->mp_state) { |
462 | case KVM_MP_STATE_RUNNABLE: |
463 | vcpu->arch.power_off = false; |
464 | break; |
465 | case KVM_MP_STATE_STOPPED: |
466 | kvm_riscv_vcpu_power_off(vcpu); |
467 | break; |
468 | default: |
469 | ret = -EINVAL; |
470 | } |
471 | |
472 | return ret; |
473 | } |
474 | |
475 | int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, |
476 | struct kvm_guest_debug *dbg) |
477 | { |
478 | /* TODO; To be implemented later. */ |
479 | return -EINVAL; |
480 | } |
481 | |
482 | static void kvm_riscv_vcpu_setup_config(struct kvm_vcpu *vcpu) |
483 | { |
484 | const unsigned long *isa = vcpu->arch.isa; |
485 | struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; |
486 | |
487 | if (riscv_isa_extension_available(isa, SVPBMT)) |
488 | cfg->henvcfg |= ENVCFG_PBMTE; |
489 | |
490 | if (riscv_isa_extension_available(isa, SSTC)) |
491 | cfg->henvcfg |= ENVCFG_STCE; |
492 | |
493 | if (riscv_isa_extension_available(isa, ZICBOM)) |
494 | cfg->henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); |
495 | |
496 | if (riscv_isa_extension_available(isa, ZICBOZ)) |
497 | cfg->henvcfg |= ENVCFG_CBZE; |
498 | |
499 | if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { |
500 | cfg->hstateen0 |= SMSTATEEN0_HSENVCFG; |
501 | if (riscv_isa_extension_available(isa, SSAIA)) |
502 | cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC | |
503 | SMSTATEEN0_AIA | |
504 | SMSTATEEN0_AIA_ISEL; |
505 | if (riscv_isa_extension_available(isa, SMSTATEEN)) |
506 | cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0; |
507 | } |
508 | } |
509 | |
510 | void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) |
511 | { |
512 | struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; |
513 | struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; |
514 | |
515 | csr_write(CSR_VSSTATUS, csr->vsstatus); |
516 | csr_write(CSR_VSIE, csr->vsie); |
517 | csr_write(CSR_VSTVEC, csr->vstvec); |
518 | csr_write(CSR_VSSCRATCH, csr->vsscratch); |
519 | csr_write(CSR_VSEPC, csr->vsepc); |
520 | csr_write(CSR_VSCAUSE, csr->vscause); |
521 | csr_write(CSR_VSTVAL, csr->vstval); |
522 | csr_write(CSR_HVIP, csr->hvip); |
523 | csr_write(CSR_VSATP, csr->vsatp); |
524 | csr_write(CSR_HENVCFG, cfg->henvcfg); |
525 | if (IS_ENABLED(CONFIG_32BIT)) |
526 | csr_write(CSR_HENVCFGH, cfg->henvcfg >> 32); |
527 | if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { |
528 | csr_write(CSR_HSTATEEN0, cfg->hstateen0); |
529 | if (IS_ENABLED(CONFIG_32BIT)) |
530 | csr_write(CSR_HSTATEEN0H, cfg->hstateen0 >> 32); |
531 | } |
532 | |
533 | kvm_riscv_gstage_update_hgatp(vcpu); |
534 | |
535 | kvm_riscv_vcpu_timer_restore(vcpu); |
536 | |
537 | kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); |
538 | kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context, |
539 | vcpu->arch.isa); |
540 | kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context); |
541 | kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, |
542 | vcpu->arch.isa); |
543 | |
544 | kvm_riscv_vcpu_aia_load(vcpu, cpu); |
545 | |
546 | kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); |
547 | |
548 | vcpu->cpu = cpu; |
549 | } |
550 | |
551 | void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) |
552 | { |
553 | struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; |
554 | |
555 | vcpu->cpu = -1; |
556 | |
557 | kvm_riscv_vcpu_aia_put(vcpu); |
558 | |
559 | kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context, |
560 | vcpu->arch.isa); |
561 | kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context); |
562 | |
563 | kvm_riscv_vcpu_timer_save(vcpu); |
564 | kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, |
565 | vcpu->arch.isa); |
566 | kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context); |
567 | |
568 | csr->vsstatus = csr_read(CSR_VSSTATUS); |
569 | csr->vsie = csr_read(CSR_VSIE); |
570 | csr->vstvec = csr_read(CSR_VSTVEC); |
571 | csr->vsscratch = csr_read(CSR_VSSCRATCH); |
572 | csr->vsepc = csr_read(CSR_VSEPC); |
573 | csr->vscause = csr_read(CSR_VSCAUSE); |
574 | csr->vstval = csr_read(CSR_VSTVAL); |
575 | csr->hvip = csr_read(CSR_HVIP); |
576 | csr->vsatp = csr_read(CSR_VSATP); |
577 | } |
578 | |
579 | static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) |
580 | { |
581 | struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); |
582 | |
583 | if (kvm_request_pending(vcpu)) { |
584 | if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) { |
585 | kvm_vcpu_srcu_read_unlock(vcpu); |
586 | rcuwait_wait_event(wait, |
587 | (!vcpu->arch.power_off) && (!vcpu->arch.pause), |
588 | TASK_INTERRUPTIBLE); |
589 | kvm_vcpu_srcu_read_lock(vcpu); |
590 | |
591 | if (vcpu->arch.power_off || vcpu->arch.pause) { |
592 | /* |
593 | * Awaken to handle a signal, request to |
594 | * sleep again later. |
595 | */ |
596 | kvm_make_request(KVM_REQ_SLEEP, vcpu); |
597 | } |
598 | } |
599 | |
600 | if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) |
601 | kvm_riscv_reset_vcpu(vcpu); |
602 | |
603 | if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu)) |
604 | kvm_riscv_gstage_update_hgatp(vcpu); |
605 | |
606 | if (kvm_check_request(KVM_REQ_FENCE_I, vcpu)) |
607 | kvm_riscv_fence_i_process(vcpu); |
608 | |
609 | /* |
610 | * The generic KVM_REQ_TLB_FLUSH is same as |
611 | * KVM_REQ_HFENCE_GVMA_VMID_ALL |
612 | */ |
613 | if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu)) |
614 | kvm_riscv_hfence_gvma_vmid_all_process(vcpu); |
615 | |
616 | if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu)) |
617 | kvm_riscv_hfence_vvma_all_process(vcpu); |
618 | |
619 | if (kvm_check_request(KVM_REQ_HFENCE, vcpu)) |
620 | kvm_riscv_hfence_process(vcpu); |
621 | |
622 | if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) |
623 | kvm_riscv_vcpu_record_steal_time(vcpu); |
624 | } |
625 | } |
626 | |
627 | static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu) |
628 | { |
629 | struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; |
630 | |
631 | csr_write(CSR_HVIP, csr->hvip); |
632 | kvm_riscv_vcpu_aia_update_hvip(vcpu); |
633 | } |
634 | |
635 | static __always_inline void kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu *vcpu) |
636 | { |
637 | struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; |
638 | struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; |
639 | struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; |
640 | |
641 | vcpu->arch.host_senvcfg = csr_swap(CSR_SENVCFG, csr->senvcfg); |
642 | if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && |
643 | (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) |
644 | vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0, |
645 | smcsr->sstateen0); |
646 | } |
647 | |
648 | static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *vcpu) |
649 | { |
650 | struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; |
651 | struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; |
652 | struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; |
653 | |
654 | csr->senvcfg = csr_swap(CSR_SENVCFG, vcpu->arch.host_senvcfg); |
655 | if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && |
656 | (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) |
657 | smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0, |
658 | vcpu->arch.host_sstateen0); |
659 | } |
660 | |
661 | /* |
662 | * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while |
663 | * the vCPU is running. |
664 | * |
665 | * This must be noinstr as instrumentation may make use of RCU, and this is not |
666 | * safe during the EQS. |
667 | */ |
668 | static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu) |
669 | { |
670 | kvm_riscv_vcpu_swap_in_guest_state(vcpu); |
671 | guest_state_enter_irqoff(); |
672 | __kvm_riscv_switch_to(&vcpu->arch); |
673 | vcpu->arch.last_exit_cpu = vcpu->cpu; |
674 | guest_state_exit_irqoff(); |
675 | kvm_riscv_vcpu_swap_in_host_state(vcpu); |
676 | } |
677 | |
678 | int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) |
679 | { |
680 | int ret; |
681 | struct kvm_cpu_trap trap; |
682 | struct kvm_run *run = vcpu->run; |
683 | |
684 | if (!vcpu->arch.ran_atleast_once) |
685 | kvm_riscv_vcpu_setup_config(vcpu); |
686 | |
687 | /* Mark this VCPU ran at least once */ |
688 | vcpu->arch.ran_atleast_once = true; |
689 | |
690 | kvm_vcpu_srcu_read_lock(vcpu); |
691 | |
692 | switch (run->exit_reason) { |
693 | case KVM_EXIT_MMIO: |
694 | /* Process MMIO value returned from user-space */ |
695 | ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run); |
696 | break; |
697 | case KVM_EXIT_RISCV_SBI: |
698 | /* Process SBI value returned from user-space */ |
699 | ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run); |
700 | break; |
701 | case KVM_EXIT_RISCV_CSR: |
702 | /* Process CSR value returned from user-space */ |
703 | ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run); |
704 | break; |
705 | default: |
706 | ret = 0; |
707 | break; |
708 | } |
709 | if (ret) { |
710 | kvm_vcpu_srcu_read_unlock(vcpu); |
711 | return ret; |
712 | } |
713 | |
714 | if (run->immediate_exit) { |
715 | kvm_vcpu_srcu_read_unlock(vcpu); |
716 | return -EINTR; |
717 | } |
718 | |
719 | vcpu_load(vcpu); |
720 | |
721 | kvm_sigset_activate(vcpu); |
722 | |
723 | ret = 1; |
724 | run->exit_reason = KVM_EXIT_UNKNOWN; |
725 | while (ret > 0) { |
726 | /* Check conditions before entering the guest */ |
727 | ret = xfer_to_guest_mode_handle_work(vcpu); |
728 | if (ret) |
729 | continue; |
730 | ret = 1; |
731 | |
732 | kvm_riscv_gstage_vmid_update(vcpu); |
733 | |
734 | kvm_riscv_check_vcpu_requests(vcpu); |
735 | |
736 | preempt_disable(); |
737 | |
738 | /* Update AIA HW state before entering guest */ |
739 | ret = kvm_riscv_vcpu_aia_update(vcpu); |
740 | if (ret <= 0) { |
741 | preempt_enable(); |
742 | continue; |
743 | } |
744 | |
745 | local_irq_disable(); |
746 | |
747 | /* |
748 | * Ensure we set mode to IN_GUEST_MODE after we disable |
749 | * interrupts and before the final VCPU requests check. |
750 | * See the comment in kvm_vcpu_exiting_guest_mode() and |
751 | * Documentation/virt/kvm/vcpu-requests.rst |
752 | */ |
753 | vcpu->mode = IN_GUEST_MODE; |
754 | |
755 | kvm_vcpu_srcu_read_unlock(vcpu); |
756 | smp_mb__after_srcu_read_unlock(); |
757 | |
758 | /* |
759 | * We might have got VCPU interrupts updated asynchronously |
760 | * so update it in HW. |
761 | */ |
762 | kvm_riscv_vcpu_flush_interrupts(vcpu); |
763 | |
764 | /* Update HVIP CSR for current CPU */ |
765 | kvm_riscv_update_hvip(vcpu); |
766 | |
767 | if (kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || |
768 | kvm_request_pending(vcpu) || |
769 | xfer_to_guest_mode_work_pending()) { |
770 | vcpu->mode = OUTSIDE_GUEST_MODE; |
771 | local_irq_enable(); |
772 | preempt_enable(); |
773 | kvm_vcpu_srcu_read_lock(vcpu); |
774 | continue; |
775 | } |
776 | |
777 | /* |
778 | * Cleanup stale TLB enteries |
779 | * |
780 | * Note: This should be done after G-stage VMID has been |
781 | * updated using kvm_riscv_gstage_vmid_ver_changed() |
782 | */ |
783 | kvm_riscv_local_tlb_sanitize(vcpu); |
784 | |
785 | guest_timing_enter_irqoff(); |
786 | |
787 | kvm_riscv_vcpu_enter_exit(vcpu); |
788 | |
789 | vcpu->mode = OUTSIDE_GUEST_MODE; |
790 | vcpu->stat.exits++; |
791 | |
792 | /* |
793 | * Save SCAUSE, STVAL, HTVAL, and HTINST because we might |
794 | * get an interrupt between __kvm_riscv_switch_to() and |
795 | * local_irq_enable() which can potentially change CSRs. |
796 | */ |
797 | trap.sepc = vcpu->arch.guest_context.sepc; |
798 | trap.scause = csr_read(CSR_SCAUSE); |
799 | trap.stval = csr_read(CSR_STVAL); |
800 | trap.htval = csr_read(CSR_HTVAL); |
801 | trap.htinst = csr_read(CSR_HTINST); |
802 | |
803 | /* Syncup interrupts state with HW */ |
804 | kvm_riscv_vcpu_sync_interrupts(vcpu); |
805 | |
806 | /* |
807 | * We must ensure that any pending interrupts are taken before |
808 | * we exit guest timing so that timer ticks are accounted as |
809 | * guest time. Transiently unmask interrupts so that any |
810 | * pending interrupts are taken. |
811 | * |
812 | * There's no barrier which ensures that pending interrupts are |
813 | * recognised, so we just hope that the CPU takes any pending |
814 | * interrupts between the enable and disable. |
815 | */ |
816 | local_irq_enable(); |
817 | local_irq_disable(); |
818 | |
819 | guest_timing_exit_irqoff(); |
820 | |
821 | local_irq_enable(); |
822 | |
823 | preempt_enable(); |
824 | |
825 | kvm_vcpu_srcu_read_lock(vcpu); |
826 | |
827 | ret = kvm_riscv_vcpu_exit(vcpu, run, &trap); |
828 | } |
829 | |
830 | kvm_sigset_deactivate(vcpu); |
831 | |
832 | vcpu_put(vcpu); |
833 | |
834 | kvm_vcpu_srcu_read_unlock(vcpu); |
835 | |
836 | return ret; |
837 | } |
838 | |