1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2019 Western Digital Corporation or its affiliates. |
4 | * |
5 | * Authors: |
6 | * Anup Patel <anup.patel@wdc.com> |
7 | */ |
8 | |
9 | #include <linux/bitops.h> |
10 | #include <linux/cpumask.h> |
11 | #include <linux/errno.h> |
12 | #include <linux/err.h> |
13 | #include <linux/module.h> |
14 | #include <linux/smp.h> |
15 | #include <linux/kvm_host.h> |
16 | #include <asm/csr.h> |
17 | |
18 | static unsigned long vmid_version = 1; |
19 | static unsigned long vmid_next; |
20 | static unsigned long vmid_bits __ro_after_init; |
21 | static DEFINE_SPINLOCK(vmid_lock); |
22 | |
23 | void __init kvm_riscv_gstage_vmid_detect(void) |
24 | { |
25 | unsigned long old; |
26 | |
27 | /* Figure-out number of VMID bits in HW */ |
28 | old = csr_read(CSR_HGATP); |
29 | csr_write(CSR_HGATP, old | HGATP_VMID); |
30 | vmid_bits = csr_read(CSR_HGATP); |
31 | vmid_bits = (vmid_bits & HGATP_VMID) >> HGATP_VMID_SHIFT; |
32 | vmid_bits = fls_long(l: vmid_bits); |
33 | csr_write(CSR_HGATP, old); |
34 | |
35 | /* We polluted local TLB so flush all guest TLB */ |
36 | kvm_riscv_local_hfence_gvma_all(); |
37 | |
38 | /* We don't use VMID bits if they are not sufficient */ |
39 | if ((1UL << vmid_bits) < num_possible_cpus()) |
40 | vmid_bits = 0; |
41 | } |
42 | |
43 | unsigned long kvm_riscv_gstage_vmid_bits(void) |
44 | { |
45 | return vmid_bits; |
46 | } |
47 | |
48 | int kvm_riscv_gstage_vmid_init(struct kvm *kvm) |
49 | { |
50 | /* Mark the initial VMID and VMID version invalid */ |
51 | kvm->arch.vmid.vmid_version = 0; |
52 | kvm->arch.vmid.vmid = 0; |
53 | |
54 | return 0; |
55 | } |
56 | |
57 | bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid) |
58 | { |
59 | if (!vmid_bits) |
60 | return false; |
61 | |
62 | return unlikely(READ_ONCE(vmid->vmid_version) != |
63 | READ_ONCE(vmid_version)); |
64 | } |
65 | |
66 | static void __local_hfence_gvma_all(void *info) |
67 | { |
68 | kvm_riscv_local_hfence_gvma_all(); |
69 | } |
70 | |
71 | void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu) |
72 | { |
73 | unsigned long i; |
74 | struct kvm_vcpu *v; |
75 | struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid; |
76 | |
77 | if (!kvm_riscv_gstage_vmid_ver_changed(vmid)) |
78 | return; |
79 | |
80 | spin_lock(lock: &vmid_lock); |
81 | |
82 | /* |
83 | * We need to re-check the vmid_version here to ensure that if |
84 | * another vcpu already allocated a valid vmid for this vm. |
85 | */ |
86 | if (!kvm_riscv_gstage_vmid_ver_changed(vmid)) { |
87 | spin_unlock(lock: &vmid_lock); |
88 | return; |
89 | } |
90 | |
91 | /* First user of a new VMID version? */ |
92 | if (unlikely(vmid_next == 0)) { |
93 | WRITE_ONCE(vmid_version, READ_ONCE(vmid_version) + 1); |
94 | vmid_next = 1; |
95 | |
96 | /* |
97 | * We ran out of VMIDs so we increment vmid_version and |
98 | * start assigning VMIDs from 1. |
99 | * |
100 | * This also means existing VMIDs assignment to all Guest |
101 | * instances is invalid and we have force VMID re-assignement |
102 | * for all Guest instances. The Guest instances that were not |
103 | * running will automatically pick-up new VMIDs because will |
104 | * call kvm_riscv_gstage_vmid_update() whenever they enter |
105 | * in-kernel run loop. For Guest instances that are already |
106 | * running, we force VM exits on all host CPUs using IPI and |
107 | * flush all Guest TLBs. |
108 | */ |
109 | on_each_cpu_mask(cpu_online_mask, func: __local_hfence_gvma_all, |
110 | NULL, wait: 1); |
111 | } |
112 | |
113 | vmid->vmid = vmid_next; |
114 | vmid_next++; |
115 | vmid_next &= (1 << vmid_bits) - 1; |
116 | |
117 | WRITE_ONCE(vmid->vmid_version, READ_ONCE(vmid_version)); |
118 | |
119 | spin_unlock(lock: &vmid_lock); |
120 | |
121 | /* Request G-stage page table update for all VCPUs */ |
122 | kvm_for_each_vcpu(i, v, vcpu->kvm) |
123 | kvm_make_request(req: KVM_REQ_UPDATE_HGATP, vcpu: v); |
124 | } |
125 | |