1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright 2019 Arm Limited |
4 | * Author: Andrew Murray <Andrew.Murray@arm.com> |
5 | */ |
6 | #include <linux/kvm_host.h> |
7 | #include <linux/perf_event.h> |
8 | |
9 | static DEFINE_PER_CPU(struct kvm_pmu_events, kvm_pmu_events); |
10 | |
11 | /* |
12 | * Given the perf event attributes and system type, determine |
13 | * if we are going to need to switch counters at guest entry/exit. |
14 | */ |
15 | static bool kvm_pmu_switch_needed(struct perf_event_attr *attr) |
16 | { |
17 | /** |
18 | * With VHE the guest kernel runs at EL1 and the host at EL2, |
19 | * where user (EL0) is excluded then we have no reason to switch |
20 | * counters. |
21 | */ |
22 | if (has_vhe() && attr->exclude_user) |
23 | return false; |
24 | |
25 | /* Only switch if attributes are different */ |
26 | return (attr->exclude_host != attr->exclude_guest); |
27 | } |
28 | |
29 | struct kvm_pmu_events *kvm_get_pmu_events(void) |
30 | { |
31 | return this_cpu_ptr(&kvm_pmu_events); |
32 | } |
33 | |
34 | /* |
35 | * Add events to track that we may want to switch at guest entry/exit |
36 | * time. |
37 | */ |
38 | void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) |
39 | { |
40 | struct kvm_pmu_events *pmu = kvm_get_pmu_events(); |
41 | |
42 | if (!kvm_arm_support_pmu_v3() || !kvm_pmu_switch_needed(attr)) |
43 | return; |
44 | |
45 | if (!attr->exclude_host) |
46 | pmu->events_host |= set; |
47 | if (!attr->exclude_guest) |
48 | pmu->events_guest |= set; |
49 | } |
50 | |
51 | /* |
52 | * Stop tracking events |
53 | */ |
54 | void kvm_clr_pmu_events(u32 clr) |
55 | { |
56 | struct kvm_pmu_events *pmu = kvm_get_pmu_events(); |
57 | |
58 | if (!kvm_arm_support_pmu_v3()) |
59 | return; |
60 | |
61 | pmu->events_host &= ~clr; |
62 | pmu->events_guest &= ~clr; |
63 | } |
64 | |
65 | #define PMEVTYPER_READ_CASE(idx) \ |
66 | case idx: \ |
67 | return read_sysreg(pmevtyper##idx##_el0) |
68 | |
69 | #define PMEVTYPER_WRITE_CASE(idx) \ |
70 | case idx: \ |
71 | write_sysreg(val, pmevtyper##idx##_el0); \ |
72 | break |
73 | |
74 | #define PMEVTYPER_CASES(readwrite) \ |
75 | PMEVTYPER_##readwrite##_CASE(0); \ |
76 | PMEVTYPER_##readwrite##_CASE(1); \ |
77 | PMEVTYPER_##readwrite##_CASE(2); \ |
78 | PMEVTYPER_##readwrite##_CASE(3); \ |
79 | PMEVTYPER_##readwrite##_CASE(4); \ |
80 | PMEVTYPER_##readwrite##_CASE(5); \ |
81 | PMEVTYPER_##readwrite##_CASE(6); \ |
82 | PMEVTYPER_##readwrite##_CASE(7); \ |
83 | PMEVTYPER_##readwrite##_CASE(8); \ |
84 | PMEVTYPER_##readwrite##_CASE(9); \ |
85 | PMEVTYPER_##readwrite##_CASE(10); \ |
86 | PMEVTYPER_##readwrite##_CASE(11); \ |
87 | PMEVTYPER_##readwrite##_CASE(12); \ |
88 | PMEVTYPER_##readwrite##_CASE(13); \ |
89 | PMEVTYPER_##readwrite##_CASE(14); \ |
90 | PMEVTYPER_##readwrite##_CASE(15); \ |
91 | PMEVTYPER_##readwrite##_CASE(16); \ |
92 | PMEVTYPER_##readwrite##_CASE(17); \ |
93 | PMEVTYPER_##readwrite##_CASE(18); \ |
94 | PMEVTYPER_##readwrite##_CASE(19); \ |
95 | PMEVTYPER_##readwrite##_CASE(20); \ |
96 | PMEVTYPER_##readwrite##_CASE(21); \ |
97 | PMEVTYPER_##readwrite##_CASE(22); \ |
98 | PMEVTYPER_##readwrite##_CASE(23); \ |
99 | PMEVTYPER_##readwrite##_CASE(24); \ |
100 | PMEVTYPER_##readwrite##_CASE(25); \ |
101 | PMEVTYPER_##readwrite##_CASE(26); \ |
102 | PMEVTYPER_##readwrite##_CASE(27); \ |
103 | PMEVTYPER_##readwrite##_CASE(28); \ |
104 | PMEVTYPER_##readwrite##_CASE(29); \ |
105 | PMEVTYPER_##readwrite##_CASE(30) |
106 | |
107 | /* |
108 | * Read a value direct from PMEVTYPER<idx> where idx is 0-30 |
109 | * or PMCCFILTR_EL0 where idx is ARMV8_PMU_CYCLE_IDX (31). |
110 | */ |
111 | static u64 kvm_vcpu_pmu_read_evtype_direct(int idx) |
112 | { |
113 | switch (idx) { |
114 | PMEVTYPER_CASES(READ); |
115 | case ARMV8_PMU_CYCLE_IDX: |
116 | return read_sysreg(pmccfiltr_el0); |
117 | default: |
118 | WARN_ON(1); |
119 | } |
120 | |
121 | return 0; |
122 | } |
123 | |
124 | /* |
125 | * Write a value direct to PMEVTYPER<idx> where idx is 0-30 |
126 | * or PMCCFILTR_EL0 where idx is ARMV8_PMU_CYCLE_IDX (31). |
127 | */ |
128 | static void kvm_vcpu_pmu_write_evtype_direct(int idx, u32 val) |
129 | { |
130 | switch (idx) { |
131 | PMEVTYPER_CASES(WRITE); |
132 | case ARMV8_PMU_CYCLE_IDX: |
133 | write_sysreg(val, pmccfiltr_el0); |
134 | break; |
135 | default: |
136 | WARN_ON(1); |
137 | } |
138 | } |
139 | |
140 | /* |
141 | * Modify ARMv8 PMU events to include EL0 counting |
142 | */ |
143 | static void kvm_vcpu_pmu_enable_el0(unsigned long events) |
144 | { |
145 | u64 typer; |
146 | u32 counter; |
147 | |
148 | for_each_set_bit(counter, &events, 32) { |
149 | typer = kvm_vcpu_pmu_read_evtype_direct(idx: counter); |
150 | typer &= ~ARMV8_PMU_EXCLUDE_EL0; |
151 | kvm_vcpu_pmu_write_evtype_direct(idx: counter, val: typer); |
152 | } |
153 | } |
154 | |
155 | /* |
156 | * Modify ARMv8 PMU events to exclude EL0 counting |
157 | */ |
158 | static void kvm_vcpu_pmu_disable_el0(unsigned long events) |
159 | { |
160 | u64 typer; |
161 | u32 counter; |
162 | |
163 | for_each_set_bit(counter, &events, 32) { |
164 | typer = kvm_vcpu_pmu_read_evtype_direct(idx: counter); |
165 | typer |= ARMV8_PMU_EXCLUDE_EL0; |
166 | kvm_vcpu_pmu_write_evtype_direct(idx: counter, val: typer); |
167 | } |
168 | } |
169 | |
170 | /* |
171 | * On VHE ensure that only guest events have EL0 counting enabled. |
172 | * This is called from both vcpu_{load,put} and the sysreg handling. |
173 | * Since the latter is preemptible, special care must be taken to |
174 | * disable preemption. |
175 | */ |
176 | void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) |
177 | { |
178 | struct kvm_pmu_events *pmu; |
179 | u32 events_guest, events_host; |
180 | |
181 | if (!kvm_arm_support_pmu_v3() || !has_vhe()) |
182 | return; |
183 | |
184 | preempt_disable(); |
185 | pmu = kvm_get_pmu_events(); |
186 | events_guest = pmu->events_guest; |
187 | events_host = pmu->events_host; |
188 | |
189 | kvm_vcpu_pmu_enable_el0(events: events_guest); |
190 | kvm_vcpu_pmu_disable_el0(events: events_host); |
191 | preempt_enable(); |
192 | } |
193 | |
194 | /* |
195 | * On VHE ensure that only host events have EL0 counting enabled |
196 | */ |
197 | void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) |
198 | { |
199 | struct kvm_pmu_events *pmu; |
200 | u32 events_guest, events_host; |
201 | |
202 | if (!kvm_arm_support_pmu_v3() || !has_vhe()) |
203 | return; |
204 | |
205 | pmu = kvm_get_pmu_events(); |
206 | events_guest = pmu->events_guest; |
207 | events_host = pmu->events_host; |
208 | |
209 | kvm_vcpu_pmu_enable_el0(events: events_host); |
210 | kvm_vcpu_pmu_disable_el0(events: events_guest); |
211 | } |
212 | |
213 | /* |
214 | * With VHE, keep track of the PMUSERENR_EL0 value for the host EL0 on the pCPU |
215 | * where PMUSERENR_EL0 for the guest is loaded, since PMUSERENR_EL0 is switched |
216 | * to the value for the guest on vcpu_load(). The value for the host EL0 |
217 | * will be restored on vcpu_put(), before returning to userspace. |
218 | * This isn't necessary for nVHE, as the register is context switched for |
219 | * every guest enter/exit. |
220 | * |
221 | * Return true if KVM takes care of the register. Otherwise return false. |
222 | */ |
223 | bool kvm_set_pmuserenr(u64 val) |
224 | { |
225 | struct kvm_cpu_context *hctxt; |
226 | struct kvm_vcpu *vcpu; |
227 | |
228 | if (!kvm_arm_support_pmu_v3() || !has_vhe()) |
229 | return false; |
230 | |
231 | vcpu = kvm_get_running_vcpu(); |
232 | if (!vcpu || !vcpu_get_flag(vcpu, PMUSERENR_ON_CPU)) |
233 | return false; |
234 | |
235 | hctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; |
236 | ctxt_sys_reg(hctxt, PMUSERENR_EL0) = val; |
237 | return true; |
238 | } |
239 | |
240 | /* |
241 | * If we interrupted the guest to update the host PMU context, make |
242 | * sure we re-apply the guest EL0 state. |
243 | */ |
244 | void kvm_vcpu_pmu_resync_el0(void) |
245 | { |
246 | struct kvm_vcpu *vcpu; |
247 | |
248 | if (!has_vhe() || !in_interrupt()) |
249 | return; |
250 | |
251 | vcpu = kvm_get_running_vcpu(); |
252 | if (!vcpu) |
253 | return; |
254 | |
255 | kvm_make_request(KVM_REQ_RESYNC_PMU_EL0, vcpu); |
256 | } |
257 | |