1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4 *
5 * Authors:
6 * Anup Patel <anup.patel@wdc.com>
7 */
8
9#include <linux/bitops.h>
10#include <linux/entry-kvm.h>
11#include <linux/errno.h>
12#include <linux/err.h>
13#include <linux/kdebug.h>
14#include <linux/module.h>
15#include <linux/percpu.h>
16#include <linux/vmalloc.h>
17#include <linux/sched/signal.h>
18#include <linux/fs.h>
19#include <linux/kvm_host.h>
20#include <asm/csr.h>
21#include <asm/cacheflush.h>
22#include <asm/kvm_vcpu_vector.h>
23
24const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
25 KVM_GENERIC_VCPU_STATS(),
26 STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
27 STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
28 STATS_DESC_COUNTER(VCPU, mmio_exit_user),
29 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
30 STATS_DESC_COUNTER(VCPU, csr_exit_user),
31 STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
32 STATS_DESC_COUNTER(VCPU, signal_exits),
33 STATS_DESC_COUNTER(VCPU, exits)
34};
35
36const struct kvm_stats_header kvm_vcpu_stats_header = {
37 .name_size = KVM_STATS_NAME_SIZE,
38 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
39 .id_offset = sizeof(struct kvm_stats_header),
40 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
41 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
42 sizeof(kvm_vcpu_stats_desc),
43};
44
45static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu)
46{
47 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
48 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;
49 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
50 struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context;
51 bool loaded;
52
53 /**
54 * The preemption should be disabled here because it races with
55 * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which
56 * also calls vcpu_load/put.
57 */
58 get_cpu();
59 loaded = (vcpu->cpu != -1);
60 if (loaded)
61 kvm_arch_vcpu_put(vcpu);
62
63 vcpu->arch.last_exit_cpu = -1;
64
65 memcpy(csr, reset_csr, sizeof(*csr));
66
67 memcpy(cntx, reset_cntx, sizeof(*cntx));
68
69 kvm_riscv_vcpu_fp_reset(vcpu);
70
71 kvm_riscv_vcpu_vector_reset(vcpu);
72
73 kvm_riscv_vcpu_timer_reset(vcpu);
74
75 kvm_riscv_vcpu_aia_reset(vcpu);
76
77 bitmap_zero(vcpu->arch.dst: irqs_pending, nbits: KVM_RISCV_VCPU_NR_IRQS);
78 bitmap_zero(dst: vcpu->arch.irqs_pending_mask, nbits: KVM_RISCV_VCPU_NR_IRQS);
79
80 kvm_riscv_vcpu_pmu_reset(vcpu);
81
82 vcpu->arch.hfence_head = 0;
83 vcpu->arch.hfence_tail = 0;
84 memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue));
85
86 kvm_riscv_vcpu_sbi_sta_reset(vcpu);
87
88 /* Reset the guest CSRs for hotplug usecase */
89 if (loaded)
90 kvm_arch_vcpu_load(vcpu, smp_processor_id());
91 put_cpu();
92}
93
94int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
95{
96 return 0;
97}
98
99int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
100{
101 int rc;
102 struct kvm_cpu_context *cntx;
103 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;
104
105 /* Mark this VCPU never ran */
106 vcpu->arch.ran_atleast_once = false;
107 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
108 bitmap_zero(dst: vcpu->arch.isa, nbits: RISCV_ISA_EXT_MAX);
109
110 /* Setup ISA features available to VCPU */
111 kvm_riscv_vcpu_setup_isa(vcpu);
112
113 /* Setup vendor, arch, and implementation details */
114 vcpu->arch.mvendorid = sbi_get_mvendorid();
115 vcpu->arch.marchid = sbi_get_marchid();
116 vcpu->arch.mimpid = sbi_get_mimpid();
117
118 /* Setup VCPU hfence queue */
119 spin_lock_init(&vcpu->arch.hfence_lock);
120
121 /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */
122 cntx = &vcpu->arch.guest_reset_context;
123 cntx->sstatus = SR_SPP | SR_SPIE;
124 cntx->hstatus = 0;
125 cntx->hstatus |= HSTATUS_VTW;
126 cntx->hstatus |= HSTATUS_SPVP;
127 cntx->hstatus |= HSTATUS_SPV;
128
129 if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx))
130 return -ENOMEM;
131
132 /* By default, make CY, TM, and IR counters accessible in VU mode */
133 reset_csr->scounteren = 0x7;
134
135 /* Setup VCPU timer */
136 kvm_riscv_vcpu_timer_init(vcpu);
137
138 /* setup performance monitoring */
139 kvm_riscv_vcpu_pmu_init(vcpu);
140
141 /* Setup VCPU AIA */
142 rc = kvm_riscv_vcpu_aia_init(vcpu);
143 if (rc)
144 return rc;
145
146 /*
147 * Setup SBI extensions
148 * NOTE: This must be the last thing to be initialized.
149 */
150 kvm_riscv_vcpu_sbi_init(vcpu);
151
152 /* Reset VCPU */
153 kvm_riscv_reset_vcpu(vcpu);
154
155 return 0;
156}
157
158void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
159{
160 /**
161 * vcpu with id 0 is the designated boot cpu.
162 * Keep all vcpus with non-zero id in power-off state so that
163 * they can be brought up using SBI HSM extension.
164 */
165 if (vcpu->vcpu_idx != 0)
166 kvm_riscv_vcpu_power_off(vcpu);
167}
168
169void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
170{
171 /* Cleanup VCPU AIA context */
172 kvm_riscv_vcpu_aia_deinit(vcpu);
173
174 /* Cleanup VCPU timer */
175 kvm_riscv_vcpu_timer_deinit(vcpu);
176
177 kvm_riscv_vcpu_pmu_deinit(vcpu);
178
179 /* Free unused pages pre-allocated for G-stage page table mappings */
180 kvm_mmu_free_memory_cache(mc: &vcpu->arch.mmu_page_cache);
181
182 /* Free vector context space for host and guest kernel */
183 kvm_riscv_vcpu_free_vector_context(vcpu);
184}
185
186int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
187{
188 return kvm_riscv_vcpu_timer_pending(vcpu);
189}
190
191void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
192{
193 kvm_riscv_aia_wakeon_hgei(vcpu, true);
194}
195
196void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
197{
198 kvm_riscv_aia_wakeon_hgei(vcpu, false);
199}
200
201int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
202{
203 return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) &&
204 !vcpu->arch.power_off && !vcpu->arch.pause);
205}
206
207int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
208{
209 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
210}
211
212bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
213{
214 return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false;
215}
216
217vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
218{
219 return VM_FAULT_SIGBUS;
220}
221
222long kvm_arch_vcpu_async_ioctl(struct file *filp,
223 unsigned int ioctl, unsigned long arg)
224{
225 struct kvm_vcpu *vcpu = filp->private_data;
226 void __user *argp = (void __user *)arg;
227
228 if (ioctl == KVM_INTERRUPT) {
229 struct kvm_interrupt irq;
230
231 if (copy_from_user(to: &irq, from: argp, n: sizeof(irq)))
232 return -EFAULT;
233
234 if (irq.irq == KVM_INTERRUPT_SET)
235 return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);
236 else
237 return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
238 }
239
240 return -ENOIOCTLCMD;
241}
242
243long kvm_arch_vcpu_ioctl(struct file *filp,
244 unsigned int ioctl, unsigned long arg)
245{
246 struct kvm_vcpu *vcpu = filp->private_data;
247 void __user *argp = (void __user *)arg;
248 long r = -EINVAL;
249
250 switch (ioctl) {
251 case KVM_SET_ONE_REG:
252 case KVM_GET_ONE_REG: {
253 struct kvm_one_reg reg;
254
255 r = -EFAULT;
256 if (copy_from_user(to: &reg, from: argp, n: sizeof(reg)))
257 break;
258
259 if (ioctl == KVM_SET_ONE_REG)
260 r = kvm_riscv_vcpu_set_reg(vcpu, &reg);
261 else
262 r = kvm_riscv_vcpu_get_reg(vcpu, &reg);
263 break;
264 }
265 case KVM_GET_REG_LIST: {
266 struct kvm_reg_list __user *user_list = argp;
267 struct kvm_reg_list reg_list;
268 unsigned int n;
269
270 r = -EFAULT;
271 if (copy_from_user(to: &reg_list, from: user_list, n: sizeof(reg_list)))
272 break;
273 n = reg_list.n;
274 reg_list.n = kvm_riscv_vcpu_num_regs(vcpu);
275 if (copy_to_user(to: user_list, from: &reg_list, n: sizeof(reg_list)))
276 break;
277 r = -E2BIG;
278 if (n < reg_list.n)
279 break;
280 r = kvm_riscv_vcpu_copy_reg_indices(vcpu, user_list->reg);
281 break;
282 }
283 default:
284 break;
285 }
286
287 return r;
288}
289
290int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
291 struct kvm_sregs *sregs)
292{
293 return -EINVAL;
294}
295
296int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
297 struct kvm_sregs *sregs)
298{
299 return -EINVAL;
300}
301
302int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
303{
304 return -EINVAL;
305}
306
307int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
308{
309 return -EINVAL;
310}
311
312int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
313 struct kvm_translation *tr)
314{
315 return -EINVAL;
316}
317
318int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
319{
320 return -EINVAL;
321}
322
323int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
324{
325 return -EINVAL;
326}
327
328void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
329{
330 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
331 unsigned long mask, val;
332
333 if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) {
334 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0);
335 val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask;
336
337 csr->hvip &= ~mask;
338 csr->hvip |= val;
339 }
340
341 /* Flush AIA high interrupts */
342 kvm_riscv_vcpu_aia_flush_interrupts(vcpu);
343}
344
345void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
346{
347 unsigned long hvip;
348 struct kvm_vcpu_arch *v = &vcpu->arch;
349 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
350
351 /* Read current HVIP and VSIE CSRs */
352 csr->vsie = csr_read(CSR_VSIE);
353
354 /* Sync-up HVIP.VSSIP bit changes does by Guest */
355 hvip = csr_read(CSR_HVIP);
356 if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) {
357 if (hvip & (1UL << IRQ_VS_SOFT)) {
358 if (!test_and_set_bit(IRQ_VS_SOFT,
359 v->irqs_pending_mask))
360 set_bit(IRQ_VS_SOFT, v->irqs_pending);
361 } else {
362 if (!test_and_set_bit(IRQ_VS_SOFT,
363 v->irqs_pending_mask))
364 clear_bit(IRQ_VS_SOFT, v->irqs_pending);
365 }
366 }
367
368 /* Sync-up AIA high interrupts */
369 kvm_riscv_vcpu_aia_sync_interrupts(vcpu);
370
371 /* Sync-up timer CSRs */
372 kvm_riscv_vcpu_timer_sync(vcpu);
373}
374
375int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
376{
377 /*
378 * We only allow VS-mode software, timer, and external
379 * interrupts when irq is one of the local interrupts
380 * defined by RISC-V privilege specification.
381 */
382 if (irq < IRQ_LOCAL_MAX &&
383 irq != IRQ_VS_SOFT &&
384 irq != IRQ_VS_TIMER &&
385 irq != IRQ_VS_EXT)
386 return -EINVAL;
387
388 set_bit(nr: irq, addr: vcpu->arch.irqs_pending);
389 smp_mb__before_atomic();
390 set_bit(nr: irq, addr: vcpu->arch.irqs_pending_mask);
391
392 kvm_vcpu_kick(vcpu);
393
394 return 0;
395}
396
397int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
398{
399 /*
400 * We only allow VS-mode software, timer, and external
401 * interrupts when irq is one of the local interrupts
402 * defined by RISC-V privilege specification.
403 */
404 if (irq < IRQ_LOCAL_MAX &&
405 irq != IRQ_VS_SOFT &&
406 irq != IRQ_VS_TIMER &&
407 irq != IRQ_VS_EXT)
408 return -EINVAL;
409
410 clear_bit(nr: irq, addr: vcpu->arch.irqs_pending);
411 smp_mb__before_atomic();
412 set_bit(nr: irq, addr: vcpu->arch.irqs_pending_mask);
413
414 return 0;
415}
416
417bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
418{
419 unsigned long ie;
420
421 ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)
422 << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask;
423 ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK &
424 (unsigned long)mask;
425 if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie)
426 return true;
427
428 /* Check AIA high interrupts */
429 return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);
430}
431
432void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
433{
434 vcpu->arch.power_off = true;
435 kvm_make_request(KVM_REQ_SLEEP, vcpu);
436 kvm_vcpu_kick(vcpu);
437}
438
439void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu)
440{
441 vcpu->arch.power_off = false;
442 kvm_vcpu_wake_up(vcpu);
443}
444
445int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
446 struct kvm_mp_state *mp_state)
447{
448 if (vcpu->arch.power_off)
449 mp_state->mp_state = KVM_MP_STATE_STOPPED;
450 else
451 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
452
453 return 0;
454}
455
456int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
457 struct kvm_mp_state *mp_state)
458{
459 int ret = 0;
460
461 switch (mp_state->mp_state) {
462 case KVM_MP_STATE_RUNNABLE:
463 vcpu->arch.power_off = false;
464 break;
465 case KVM_MP_STATE_STOPPED:
466 kvm_riscv_vcpu_power_off(vcpu);
467 break;
468 default:
469 ret = -EINVAL;
470 }
471
472 return ret;
473}
474
475int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
476 struct kvm_guest_debug *dbg)
477{
478 /* TODO; To be implemented later. */
479 return -EINVAL;
480}
481
482static void kvm_riscv_vcpu_setup_config(struct kvm_vcpu *vcpu)
483{
484 const unsigned long *isa = vcpu->arch.isa;
485 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg;
486
487 if (riscv_isa_extension_available(isa, SVPBMT))
488 cfg->henvcfg |= ENVCFG_PBMTE;
489
490 if (riscv_isa_extension_available(isa, SSTC))
491 cfg->henvcfg |= ENVCFG_STCE;
492
493 if (riscv_isa_extension_available(isa, ZICBOM))
494 cfg->henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE);
495
496 if (riscv_isa_extension_available(isa, ZICBOZ))
497 cfg->henvcfg |= ENVCFG_CBZE;
498
499 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) {
500 cfg->hstateen0 |= SMSTATEEN0_HSENVCFG;
501 if (riscv_isa_extension_available(isa, SSAIA))
502 cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC |
503 SMSTATEEN0_AIA |
504 SMSTATEEN0_AIA_ISEL;
505 if (riscv_isa_extension_available(isa, SMSTATEEN))
506 cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0;
507 }
508}
509
510void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
511{
512 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
513 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg;
514
515 csr_write(CSR_VSSTATUS, csr->vsstatus);
516 csr_write(CSR_VSIE, csr->vsie);
517 csr_write(CSR_VSTVEC, csr->vstvec);
518 csr_write(CSR_VSSCRATCH, csr->vsscratch);
519 csr_write(CSR_VSEPC, csr->vsepc);
520 csr_write(CSR_VSCAUSE, csr->vscause);
521 csr_write(CSR_VSTVAL, csr->vstval);
522 csr_write(CSR_HVIP, csr->hvip);
523 csr_write(CSR_VSATP, csr->vsatp);
524 csr_write(CSR_HENVCFG, cfg->henvcfg);
525 if (IS_ENABLED(CONFIG_32BIT))
526 csr_write(CSR_HENVCFGH, cfg->henvcfg >> 32);
527 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) {
528 csr_write(CSR_HSTATEEN0, cfg->hstateen0);
529 if (IS_ENABLED(CONFIG_32BIT))
530 csr_write(CSR_HSTATEEN0H, cfg->hstateen0 >> 32);
531 }
532
533 kvm_riscv_gstage_update_hgatp(vcpu);
534
535 kvm_riscv_vcpu_timer_restore(vcpu);
536
537 kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context);
538 kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context,
539 vcpu->arch.isa);
540 kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context);
541 kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context,
542 vcpu->arch.isa);
543
544 kvm_riscv_vcpu_aia_load(vcpu, cpu);
545
546 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
547
548 vcpu->cpu = cpu;
549}
550
551void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
552{
553 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
554
555 vcpu->cpu = -1;
556
557 kvm_riscv_vcpu_aia_put(vcpu);
558
559 kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context,
560 vcpu->arch.isa);
561 kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context);
562
563 kvm_riscv_vcpu_timer_save(vcpu);
564 kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context,
565 vcpu->arch.isa);
566 kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context);
567
568 csr->vsstatus = csr_read(CSR_VSSTATUS);
569 csr->vsie = csr_read(CSR_VSIE);
570 csr->vstvec = csr_read(CSR_VSTVEC);
571 csr->vsscratch = csr_read(CSR_VSSCRATCH);
572 csr->vsepc = csr_read(CSR_VSEPC);
573 csr->vscause = csr_read(CSR_VSCAUSE);
574 csr->vstval = csr_read(CSR_VSTVAL);
575 csr->hvip = csr_read(CSR_HVIP);
576 csr->vsatp = csr_read(CSR_VSATP);
577}
578
579static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
580{
581 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
582
583 if (kvm_request_pending(vcpu)) {
584 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) {
585 kvm_vcpu_srcu_read_unlock(vcpu);
586 rcuwait_wait_event(wait,
587 (!vcpu->arch.power_off) && (!vcpu->arch.pause),
588 TASK_INTERRUPTIBLE);
589 kvm_vcpu_srcu_read_lock(vcpu);
590
591 if (vcpu->arch.power_off || vcpu->arch.pause) {
592 /*
593 * Awaken to handle a signal, request to
594 * sleep again later.
595 */
596 kvm_make_request(KVM_REQ_SLEEP, vcpu);
597 }
598 }
599
600 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
601 kvm_riscv_reset_vcpu(vcpu);
602
603 if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))
604 kvm_riscv_gstage_update_hgatp(vcpu);
605
606 if (kvm_check_request(KVM_REQ_FENCE_I, vcpu))
607 kvm_riscv_fence_i_process(vcpu);
608
609 /*
610 * The generic KVM_REQ_TLB_FLUSH is same as
611 * KVM_REQ_HFENCE_GVMA_VMID_ALL
612 */
613 if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu))
614 kvm_riscv_hfence_gvma_vmid_all_process(vcpu);
615
616 if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu))
617 kvm_riscv_hfence_vvma_all_process(vcpu);
618
619 if (kvm_check_request(KVM_REQ_HFENCE, vcpu))
620 kvm_riscv_hfence_process(vcpu);
621
622 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
623 kvm_riscv_vcpu_record_steal_time(vcpu);
624 }
625}
626
627static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu)
628{
629 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
630
631 csr_write(CSR_HVIP, csr->hvip);
632 kvm_riscv_vcpu_aia_update_hvip(vcpu);
633}
634
635static __always_inline void kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu *vcpu)
636{
637 struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr;
638 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
639 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg;
640
641 vcpu->arch.host_senvcfg = csr_swap(CSR_SENVCFG, csr->senvcfg);
642 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) &&
643 (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0))
644 vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0,
645 smcsr->sstateen0);
646}
647
648static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *vcpu)
649{
650 struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr;
651 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
652 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg;
653
654 csr->senvcfg = csr_swap(CSR_SENVCFG, vcpu->arch.host_senvcfg);
655 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) &&
656 (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0))
657 smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0,
658 vcpu->arch.host_sstateen0);
659}
660
661/*
662 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
663 * the vCPU is running.
664 *
665 * This must be noinstr as instrumentation may make use of RCU, and this is not
666 * safe during the EQS.
667 */
668static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu)
669{
670 kvm_riscv_vcpu_swap_in_guest_state(vcpu);
671 guest_state_enter_irqoff();
672 __kvm_riscv_switch_to(&vcpu->arch);
673 vcpu->arch.last_exit_cpu = vcpu->cpu;
674 guest_state_exit_irqoff();
675 kvm_riscv_vcpu_swap_in_host_state(vcpu);
676}
677
678int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
679{
680 int ret;
681 struct kvm_cpu_trap trap;
682 struct kvm_run *run = vcpu->run;
683
684 if (!vcpu->arch.ran_atleast_once)
685 kvm_riscv_vcpu_setup_config(vcpu);
686
687 /* Mark this VCPU ran at least once */
688 vcpu->arch.ran_atleast_once = true;
689
690 kvm_vcpu_srcu_read_lock(vcpu);
691
692 switch (run->exit_reason) {
693 case KVM_EXIT_MMIO:
694 /* Process MMIO value returned from user-space */
695 ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run);
696 break;
697 case KVM_EXIT_RISCV_SBI:
698 /* Process SBI value returned from user-space */
699 ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run);
700 break;
701 case KVM_EXIT_RISCV_CSR:
702 /* Process CSR value returned from user-space */
703 ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run);
704 break;
705 default:
706 ret = 0;
707 break;
708 }
709 if (ret) {
710 kvm_vcpu_srcu_read_unlock(vcpu);
711 return ret;
712 }
713
714 if (run->immediate_exit) {
715 kvm_vcpu_srcu_read_unlock(vcpu);
716 return -EINTR;
717 }
718
719 vcpu_load(vcpu);
720
721 kvm_sigset_activate(vcpu);
722
723 ret = 1;
724 run->exit_reason = KVM_EXIT_UNKNOWN;
725 while (ret > 0) {
726 /* Check conditions before entering the guest */
727 ret = xfer_to_guest_mode_handle_work(vcpu);
728 if (ret)
729 continue;
730 ret = 1;
731
732 kvm_riscv_gstage_vmid_update(vcpu);
733
734 kvm_riscv_check_vcpu_requests(vcpu);
735
736 preempt_disable();
737
738 /* Update AIA HW state before entering guest */
739 ret = kvm_riscv_vcpu_aia_update(vcpu);
740 if (ret <= 0) {
741 preempt_enable();
742 continue;
743 }
744
745 local_irq_disable();
746
747 /*
748 * Ensure we set mode to IN_GUEST_MODE after we disable
749 * interrupts and before the final VCPU requests check.
750 * See the comment in kvm_vcpu_exiting_guest_mode() and
751 * Documentation/virt/kvm/vcpu-requests.rst
752 */
753 vcpu->mode = IN_GUEST_MODE;
754
755 kvm_vcpu_srcu_read_unlock(vcpu);
756 smp_mb__after_srcu_read_unlock();
757
758 /*
759 * We might have got VCPU interrupts updated asynchronously
760 * so update it in HW.
761 */
762 kvm_riscv_vcpu_flush_interrupts(vcpu);
763
764 /* Update HVIP CSR for current CPU */
765 kvm_riscv_update_hvip(vcpu);
766
767 if (kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
768 kvm_request_pending(vcpu) ||
769 xfer_to_guest_mode_work_pending()) {
770 vcpu->mode = OUTSIDE_GUEST_MODE;
771 local_irq_enable();
772 preempt_enable();
773 kvm_vcpu_srcu_read_lock(vcpu);
774 continue;
775 }
776
777 /*
778 * Cleanup stale TLB enteries
779 *
780 * Note: This should be done after G-stage VMID has been
781 * updated using kvm_riscv_gstage_vmid_ver_changed()
782 */
783 kvm_riscv_local_tlb_sanitize(vcpu);
784
785 guest_timing_enter_irqoff();
786
787 kvm_riscv_vcpu_enter_exit(vcpu);
788
789 vcpu->mode = OUTSIDE_GUEST_MODE;
790 vcpu->stat.exits++;
791
792 /*
793 * Save SCAUSE, STVAL, HTVAL, and HTINST because we might
794 * get an interrupt between __kvm_riscv_switch_to() and
795 * local_irq_enable() which can potentially change CSRs.
796 */
797 trap.sepc = vcpu->arch.guest_context.sepc;
798 trap.scause = csr_read(CSR_SCAUSE);
799 trap.stval = csr_read(CSR_STVAL);
800 trap.htval = csr_read(CSR_HTVAL);
801 trap.htinst = csr_read(CSR_HTINST);
802
803 /* Syncup interrupts state with HW */
804 kvm_riscv_vcpu_sync_interrupts(vcpu);
805
806 /*
807 * We must ensure that any pending interrupts are taken before
808 * we exit guest timing so that timer ticks are accounted as
809 * guest time. Transiently unmask interrupts so that any
810 * pending interrupts are taken.
811 *
812 * There's no barrier which ensures that pending interrupts are
813 * recognised, so we just hope that the CPU takes any pending
814 * interrupts between the enable and disable.
815 */
816 local_irq_enable();
817 local_irq_disable();
818
819 guest_timing_exit_irqoff();
820
821 local_irq_enable();
822
823 preempt_enable();
824
825 kvm_vcpu_srcu_read_lock(vcpu);
826
827 ret = kvm_riscv_vcpu_exit(vcpu, run, &trap);
828 }
829
830 kvm_sigset_deactivate(vcpu);
831
832 vcpu_put(vcpu);
833
834 kvm_vcpu_srcu_read_unlock(vcpu);
835
836 return ret;
837}
838

source code of linux/arch/riscv/kvm/vcpu.c