1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2023 Rivos Inc |
4 | * |
5 | * Authors: |
6 | * Atish Patra <atishp@rivosinc.com> |
7 | */ |
8 | |
9 | #define pr_fmt(fmt) "riscv-kvm-pmu: " fmt |
10 | #include <linux/errno.h> |
11 | #include <linux/err.h> |
12 | #include <linux/kvm_host.h> |
13 | #include <linux/perf/riscv_pmu.h> |
14 | #include <asm/csr.h> |
15 | #include <asm/kvm_vcpu_sbi.h> |
16 | #include <asm/kvm_vcpu_pmu.h> |
17 | #include <linux/bitops.h> |
18 | |
19 | #define kvm_pmu_num_counters(pmu) ((pmu)->num_hw_ctrs + (pmu)->num_fw_ctrs) |
20 | #define get_event_type(x) (((x) & SBI_PMU_EVENT_IDX_TYPE_MASK) >> 16) |
21 | #define get_event_code(x) ((x) & SBI_PMU_EVENT_IDX_CODE_MASK) |
22 | |
23 | static enum perf_hw_id hw_event_perf_map[SBI_PMU_HW_GENERAL_MAX] = { |
24 | [SBI_PMU_HW_CPU_CYCLES] = PERF_COUNT_HW_CPU_CYCLES, |
25 | [SBI_PMU_HW_INSTRUCTIONS] = PERF_COUNT_HW_INSTRUCTIONS, |
26 | [SBI_PMU_HW_CACHE_REFERENCES] = PERF_COUNT_HW_CACHE_REFERENCES, |
27 | [SBI_PMU_HW_CACHE_MISSES] = PERF_COUNT_HW_CACHE_MISSES, |
28 | [SBI_PMU_HW_BRANCH_INSTRUCTIONS] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS, |
29 | [SBI_PMU_HW_BRANCH_MISSES] = PERF_COUNT_HW_BRANCH_MISSES, |
30 | [SBI_PMU_HW_BUS_CYCLES] = PERF_COUNT_HW_BUS_CYCLES, |
31 | [SBI_PMU_HW_STALLED_CYCLES_FRONTEND] = PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, |
32 | [SBI_PMU_HW_STALLED_CYCLES_BACKEND] = PERF_COUNT_HW_STALLED_CYCLES_BACKEND, |
33 | [SBI_PMU_HW_REF_CPU_CYCLES] = PERF_COUNT_HW_REF_CPU_CYCLES, |
34 | }; |
35 | |
36 | static u64 kvm_pmu_get_sample_period(struct kvm_pmc *pmc) |
37 | { |
38 | u64 counter_val_mask = GENMASK(pmc->cinfo.width, 0); |
39 | u64 sample_period; |
40 | |
41 | if (!pmc->counter_val) |
42 | sample_period = counter_val_mask + 1; |
43 | else |
44 | sample_period = (-pmc->counter_val) & counter_val_mask; |
45 | |
46 | return sample_period; |
47 | } |
48 | |
49 | static u32 kvm_pmu_get_perf_event_type(unsigned long eidx) |
50 | { |
51 | enum sbi_pmu_event_type etype = get_event_type(eidx); |
52 | u32 type = PERF_TYPE_MAX; |
53 | |
54 | switch (etype) { |
55 | case SBI_PMU_EVENT_TYPE_HW: |
56 | type = PERF_TYPE_HARDWARE; |
57 | break; |
58 | case SBI_PMU_EVENT_TYPE_CACHE: |
59 | type = PERF_TYPE_HW_CACHE; |
60 | break; |
61 | case SBI_PMU_EVENT_TYPE_RAW: |
62 | case SBI_PMU_EVENT_TYPE_FW: |
63 | type = PERF_TYPE_RAW; |
64 | break; |
65 | default: |
66 | break; |
67 | } |
68 | |
69 | return type; |
70 | } |
71 | |
72 | static bool kvm_pmu_is_fw_event(unsigned long eidx) |
73 | { |
74 | return get_event_type(eidx) == SBI_PMU_EVENT_TYPE_FW; |
75 | } |
76 | |
77 | static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc) |
78 | { |
79 | if (pmc->perf_event) { |
80 | perf_event_disable(event: pmc->perf_event); |
81 | perf_event_release_kernel(event: pmc->perf_event); |
82 | pmc->perf_event = NULL; |
83 | } |
84 | } |
85 | |
86 | static u64 kvm_pmu_get_perf_event_hw_config(u32 sbi_event_code) |
87 | { |
88 | return hw_event_perf_map[sbi_event_code]; |
89 | } |
90 | |
91 | static u64 kvm_pmu_get_perf_event_cache_config(u32 sbi_event_code) |
92 | { |
93 | u64 config = U64_MAX; |
94 | unsigned int cache_type, cache_op, cache_result; |
95 | |
96 | /* All the cache event masks lie within 0xFF. No separate masking is necessary */ |
97 | cache_type = (sbi_event_code & SBI_PMU_EVENT_CACHE_ID_CODE_MASK) >> |
98 | SBI_PMU_EVENT_CACHE_ID_SHIFT; |
99 | cache_op = (sbi_event_code & SBI_PMU_EVENT_CACHE_OP_ID_CODE_MASK) >> |
100 | SBI_PMU_EVENT_CACHE_OP_SHIFT; |
101 | cache_result = sbi_event_code & SBI_PMU_EVENT_CACHE_RESULT_ID_CODE_MASK; |
102 | |
103 | if (cache_type >= PERF_COUNT_HW_CACHE_MAX || |
104 | cache_op >= PERF_COUNT_HW_CACHE_OP_MAX || |
105 | cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
106 | return config; |
107 | |
108 | config = cache_type | (cache_op << 8) | (cache_result << 16); |
109 | |
110 | return config; |
111 | } |
112 | |
113 | static u64 kvm_pmu_get_perf_event_config(unsigned long eidx, uint64_t evt_data) |
114 | { |
115 | enum sbi_pmu_event_type etype = get_event_type(eidx); |
116 | u32 ecode = get_event_code(eidx); |
117 | u64 config = U64_MAX; |
118 | |
119 | switch (etype) { |
120 | case SBI_PMU_EVENT_TYPE_HW: |
121 | if (ecode < SBI_PMU_HW_GENERAL_MAX) |
122 | config = kvm_pmu_get_perf_event_hw_config(sbi_event_code: ecode); |
123 | break; |
124 | case SBI_PMU_EVENT_TYPE_CACHE: |
125 | config = kvm_pmu_get_perf_event_cache_config(sbi_event_code: ecode); |
126 | break; |
127 | case SBI_PMU_EVENT_TYPE_RAW: |
128 | config = evt_data & RISCV_PMU_RAW_EVENT_MASK; |
129 | break; |
130 | case SBI_PMU_EVENT_TYPE_FW: |
131 | if (ecode < SBI_PMU_FW_MAX) |
132 | config = (1ULL << 63) | ecode; |
133 | break; |
134 | default: |
135 | break; |
136 | } |
137 | |
138 | return config; |
139 | } |
140 | |
141 | static int kvm_pmu_get_fixed_pmc_index(unsigned long eidx) |
142 | { |
143 | u32 etype = kvm_pmu_get_perf_event_type(eidx); |
144 | u32 ecode = get_event_code(eidx); |
145 | |
146 | if (etype != SBI_PMU_EVENT_TYPE_HW) |
147 | return -EINVAL; |
148 | |
149 | if (ecode == SBI_PMU_HW_CPU_CYCLES) |
150 | return 0; |
151 | else if (ecode == SBI_PMU_HW_INSTRUCTIONS) |
152 | return 2; |
153 | else |
154 | return -EINVAL; |
155 | } |
156 | |
157 | static int kvm_pmu_get_programmable_pmc_index(struct kvm_pmu *kvpmu, unsigned long eidx, |
158 | unsigned long cbase, unsigned long cmask) |
159 | { |
160 | int ctr_idx = -1; |
161 | int i, pmc_idx; |
162 | int min, max; |
163 | |
164 | if (kvm_pmu_is_fw_event(eidx)) { |
165 | /* Firmware counters are mapped 1:1 starting from num_hw_ctrs for simplicity */ |
166 | min = kvpmu->num_hw_ctrs; |
167 | max = min + kvpmu->num_fw_ctrs; |
168 | } else { |
169 | /* First 3 counters are reserved for fixed counters */ |
170 | min = 3; |
171 | max = kvpmu->num_hw_ctrs; |
172 | } |
173 | |
174 | for_each_set_bit(i, &cmask, BITS_PER_LONG) { |
175 | pmc_idx = i + cbase; |
176 | if ((pmc_idx >= min && pmc_idx < max) && |
177 | !test_bit(pmc_idx, kvpmu->pmc_in_use)) { |
178 | ctr_idx = pmc_idx; |
179 | break; |
180 | } |
181 | } |
182 | |
183 | return ctr_idx; |
184 | } |
185 | |
186 | static int pmu_get_pmc_index(struct kvm_pmu *pmu, unsigned long eidx, |
187 | unsigned long cbase, unsigned long cmask) |
188 | { |
189 | int ret; |
190 | |
191 | /* Fixed counters need to be have fixed mapping as they have different width */ |
192 | ret = kvm_pmu_get_fixed_pmc_index(eidx); |
193 | if (ret >= 0) |
194 | return ret; |
195 | |
196 | return kvm_pmu_get_programmable_pmc_index(kvpmu: pmu, eidx, cbase, cmask); |
197 | } |
198 | |
199 | static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, |
200 | unsigned long *out_val) |
201 | { |
202 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
203 | struct kvm_pmc *pmc; |
204 | u64 enabled, running; |
205 | int fevent_code; |
206 | |
207 | pmc = &kvpmu->pmc[cidx]; |
208 | |
209 | if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) { |
210 | fevent_code = get_event_code(pmc->event_idx); |
211 | pmc->counter_val = kvpmu->fw_event[fevent_code].value; |
212 | } else if (pmc->perf_event) { |
213 | pmc->counter_val += perf_event_read_value(event: pmc->perf_event, enabled: &enabled, running: &running); |
214 | } else { |
215 | return -EINVAL; |
216 | } |
217 | *out_val = pmc->counter_val; |
218 | |
219 | return 0; |
220 | } |
221 | |
222 | static int kvm_pmu_validate_counter_mask(struct kvm_pmu *kvpmu, unsigned long ctr_base, |
223 | unsigned long ctr_mask) |
224 | { |
225 | /* Make sure the we have a valid counter mask requested from the caller */ |
226 | if (!ctr_mask || (ctr_base + __fls(word: ctr_mask) >= kvm_pmu_num_counters(kvpmu))) |
227 | return -EINVAL; |
228 | |
229 | return 0; |
230 | } |
231 | |
232 | static int kvm_pmu_create_perf_event(struct kvm_pmc *pmc, struct perf_event_attr *attr, |
233 | unsigned long flags, unsigned long eidx, unsigned long evtdata) |
234 | { |
235 | struct perf_event *event; |
236 | |
237 | kvm_pmu_release_perf_event(pmc); |
238 | attr->config = kvm_pmu_get_perf_event_config(eidx, evt_data: evtdata); |
239 | if (flags & SBI_PMU_CFG_FLAG_CLEAR_VALUE) { |
240 | //TODO: Do we really want to clear the value in hardware counter |
241 | pmc->counter_val = 0; |
242 | } |
243 | |
244 | /* |
245 | * Set the default sample_period for now. The guest specified value |
246 | * will be updated in the start call. |
247 | */ |
248 | attr->sample_period = kvm_pmu_get_sample_period(pmc); |
249 | |
250 | event = perf_event_create_kernel_counter(attr, cpu: -1, current, NULL, context: pmc); |
251 | if (IS_ERR(ptr: event)) { |
252 | pr_err("kvm pmu event creation failed for eidx %lx: %ld\n" , eidx, PTR_ERR(event)); |
253 | return PTR_ERR(ptr: event); |
254 | } |
255 | |
256 | pmc->perf_event = event; |
257 | if (flags & SBI_PMU_CFG_FLAG_AUTO_START) |
258 | perf_event_enable(event: pmc->perf_event); |
259 | |
260 | return 0; |
261 | } |
262 | |
263 | int kvm_riscv_vcpu_pmu_incr_fw(struct kvm_vcpu *vcpu, unsigned long fid) |
264 | { |
265 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
266 | struct kvm_fw_event *fevent; |
267 | |
268 | if (!kvpmu || fid >= SBI_PMU_FW_MAX) |
269 | return -EINVAL; |
270 | |
271 | fevent = &kvpmu->fw_event[fid]; |
272 | if (fevent->started) |
273 | fevent->value++; |
274 | |
275 | return 0; |
276 | } |
277 | |
278 | int kvm_riscv_vcpu_pmu_read_hpm(struct kvm_vcpu *vcpu, unsigned int csr_num, |
279 | unsigned long *val, unsigned long new_val, |
280 | unsigned long wr_mask) |
281 | { |
282 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
283 | int cidx, ret = KVM_INSN_CONTINUE_NEXT_SEPC; |
284 | |
285 | if (!kvpmu || !kvpmu->init_done) { |
286 | /* |
287 | * In absence of sscofpmf in the platform, the guest OS may use |
288 | * the legacy PMU driver to read cycle/instret. In that case, |
289 | * just return 0 to avoid any illegal trap. However, any other |
290 | * hpmcounter access should result in illegal trap as they must |
291 | * be access through SBI PMU only. |
292 | */ |
293 | if (csr_num == CSR_CYCLE || csr_num == CSR_INSTRET) { |
294 | *val = 0; |
295 | return ret; |
296 | } else { |
297 | return KVM_INSN_ILLEGAL_TRAP; |
298 | } |
299 | } |
300 | |
301 | /* The counter CSR are read only. Thus, any write should result in illegal traps */ |
302 | if (wr_mask) |
303 | return KVM_INSN_ILLEGAL_TRAP; |
304 | |
305 | cidx = csr_num - CSR_CYCLE; |
306 | |
307 | if (pmu_ctr_read(vcpu, cidx, val) < 0) |
308 | return KVM_INSN_ILLEGAL_TRAP; |
309 | |
310 | return ret; |
311 | } |
312 | |
313 | int kvm_riscv_vcpu_pmu_num_ctrs(struct kvm_vcpu *vcpu, |
314 | struct kvm_vcpu_sbi_return *retdata) |
315 | { |
316 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
317 | |
318 | retdata->out_val = kvm_pmu_num_counters(kvpmu); |
319 | |
320 | return 0; |
321 | } |
322 | |
323 | int kvm_riscv_vcpu_pmu_ctr_info(struct kvm_vcpu *vcpu, unsigned long cidx, |
324 | struct kvm_vcpu_sbi_return *retdata) |
325 | { |
326 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
327 | |
328 | if (cidx > RISCV_KVM_MAX_COUNTERS || cidx == 1) { |
329 | retdata->err_val = SBI_ERR_INVALID_PARAM; |
330 | return 0; |
331 | } |
332 | |
333 | retdata->out_val = kvpmu->pmc[cidx].cinfo.value; |
334 | |
335 | return 0; |
336 | } |
337 | |
338 | int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base, |
339 | unsigned long ctr_mask, unsigned long flags, u64 ival, |
340 | struct kvm_vcpu_sbi_return *retdata) |
341 | { |
342 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
343 | int i, pmc_index, sbiret = 0; |
344 | struct kvm_pmc *pmc; |
345 | int fevent_code; |
346 | |
347 | if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) { |
348 | sbiret = SBI_ERR_INVALID_PARAM; |
349 | goto out; |
350 | } |
351 | |
352 | /* Start the counters that have been configured and requested by the guest */ |
353 | for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { |
354 | pmc_index = i + ctr_base; |
355 | if (!test_bit(pmc_index, kvpmu->pmc_in_use)) |
356 | continue; |
357 | pmc = &kvpmu->pmc[pmc_index]; |
358 | if (flags & SBI_PMU_START_FLAG_SET_INIT_VALUE) |
359 | pmc->counter_val = ival; |
360 | if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) { |
361 | fevent_code = get_event_code(pmc->event_idx); |
362 | if (fevent_code >= SBI_PMU_FW_MAX) { |
363 | sbiret = SBI_ERR_INVALID_PARAM; |
364 | goto out; |
365 | } |
366 | |
367 | /* Check if the counter was already started for some reason */ |
368 | if (kvpmu->fw_event[fevent_code].started) { |
369 | sbiret = SBI_ERR_ALREADY_STARTED; |
370 | continue; |
371 | } |
372 | |
373 | kvpmu->fw_event[fevent_code].started = true; |
374 | kvpmu->fw_event[fevent_code].value = pmc->counter_val; |
375 | } else if (pmc->perf_event) { |
376 | if (unlikely(pmc->started)) { |
377 | sbiret = SBI_ERR_ALREADY_STARTED; |
378 | continue; |
379 | } |
380 | perf_event_period(pmc->perf_event, kvm_pmu_get_sample_period(pmc)); |
381 | perf_event_enable(pmc->perf_event); |
382 | pmc->started = true; |
383 | } else { |
384 | sbiret = SBI_ERR_INVALID_PARAM; |
385 | } |
386 | } |
387 | |
388 | out: |
389 | retdata->err_val = sbiret; |
390 | |
391 | return 0; |
392 | } |
393 | |
394 | int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, |
395 | unsigned long ctr_mask, unsigned long flags, |
396 | struct kvm_vcpu_sbi_return *retdata) |
397 | { |
398 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
399 | int i, pmc_index, sbiret = 0; |
400 | u64 enabled, running; |
401 | struct kvm_pmc *pmc; |
402 | int fevent_code; |
403 | |
404 | if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) { |
405 | sbiret = SBI_ERR_INVALID_PARAM; |
406 | goto out; |
407 | } |
408 | |
409 | /* Stop the counters that have been configured and requested by the guest */ |
410 | for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) { |
411 | pmc_index = i + ctr_base; |
412 | if (!test_bit(pmc_index, kvpmu->pmc_in_use)) |
413 | continue; |
414 | pmc = &kvpmu->pmc[pmc_index]; |
415 | if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) { |
416 | fevent_code = get_event_code(pmc->event_idx); |
417 | if (fevent_code >= SBI_PMU_FW_MAX) { |
418 | sbiret = SBI_ERR_INVALID_PARAM; |
419 | goto out; |
420 | } |
421 | |
422 | if (!kvpmu->fw_event[fevent_code].started) |
423 | sbiret = SBI_ERR_ALREADY_STOPPED; |
424 | |
425 | kvpmu->fw_event[fevent_code].started = false; |
426 | } else if (pmc->perf_event) { |
427 | if (pmc->started) { |
428 | /* Stop counting the counter */ |
429 | perf_event_disable(pmc->perf_event); |
430 | pmc->started = false; |
431 | } else { |
432 | sbiret = SBI_ERR_ALREADY_STOPPED; |
433 | } |
434 | |
435 | if (flags & SBI_PMU_STOP_FLAG_RESET) { |
436 | /* Relase the counter if this is a reset request */ |
437 | pmc->counter_val += perf_event_read_value(pmc->perf_event, |
438 | &enabled, &running); |
439 | kvm_pmu_release_perf_event(pmc); |
440 | } |
441 | } else { |
442 | sbiret = SBI_ERR_INVALID_PARAM; |
443 | } |
444 | if (flags & SBI_PMU_STOP_FLAG_RESET) { |
445 | pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID; |
446 | clear_bit(pmc_index, kvpmu->pmc_in_use); |
447 | } |
448 | } |
449 | |
450 | out: |
451 | retdata->err_val = sbiret; |
452 | |
453 | return 0; |
454 | } |
455 | |
456 | int kvm_riscv_vcpu_pmu_ctr_cfg_match(struct kvm_vcpu *vcpu, unsigned long ctr_base, |
457 | unsigned long ctr_mask, unsigned long flags, |
458 | unsigned long eidx, u64 evtdata, |
459 | struct kvm_vcpu_sbi_return *retdata) |
460 | { |
461 | int ctr_idx, ret, sbiret = 0; |
462 | bool is_fevent; |
463 | unsigned long event_code; |
464 | u32 etype = kvm_pmu_get_perf_event_type(eidx); |
465 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
466 | struct kvm_pmc *pmc = NULL; |
467 | struct perf_event_attr attr = { |
468 | .type = etype, |
469 | .size = sizeof(struct perf_event_attr), |
470 | .pinned = true, |
471 | /* |
472 | * It should never reach here if the platform doesn't support the sscofpmf |
473 | * extension as mode filtering won't work without it. |
474 | */ |
475 | .exclude_host = true, |
476 | .exclude_hv = true, |
477 | .exclude_user = !!(flags & SBI_PMU_CFG_FLAG_SET_UINH), |
478 | .exclude_kernel = !!(flags & SBI_PMU_CFG_FLAG_SET_SINH), |
479 | .config1 = RISCV_PMU_CONFIG1_GUEST_EVENTS, |
480 | }; |
481 | |
482 | if (kvm_pmu_validate_counter_mask(kvpmu, ctr_base, ctr_mask) < 0) { |
483 | sbiret = SBI_ERR_INVALID_PARAM; |
484 | goto out; |
485 | } |
486 | |
487 | event_code = get_event_code(eidx); |
488 | is_fevent = kvm_pmu_is_fw_event(eidx); |
489 | if (is_fevent && event_code >= SBI_PMU_FW_MAX) { |
490 | sbiret = SBI_ERR_NOT_SUPPORTED; |
491 | goto out; |
492 | } |
493 | |
494 | /* |
495 | * SKIP_MATCH flag indicates the caller is aware of the assigned counter |
496 | * for this event. Just do a sanity check if it already marked used. |
497 | */ |
498 | if (flags & SBI_PMU_CFG_FLAG_SKIP_MATCH) { |
499 | if (!test_bit(ctr_base + __ffs(ctr_mask), kvpmu->pmc_in_use)) { |
500 | sbiret = SBI_ERR_FAILURE; |
501 | goto out; |
502 | } |
503 | ctr_idx = ctr_base + __ffs(ctr_mask); |
504 | } else { |
505 | ctr_idx = pmu_get_pmc_index(pmu: kvpmu, eidx, cbase: ctr_base, cmask: ctr_mask); |
506 | if (ctr_idx < 0) { |
507 | sbiret = SBI_ERR_NOT_SUPPORTED; |
508 | goto out; |
509 | } |
510 | } |
511 | |
512 | pmc = &kvpmu->pmc[ctr_idx]; |
513 | pmc->idx = ctr_idx; |
514 | |
515 | if (is_fevent) { |
516 | if (flags & SBI_PMU_CFG_FLAG_AUTO_START) |
517 | kvpmu->fw_event[event_code].started = true; |
518 | } else { |
519 | ret = kvm_pmu_create_perf_event(pmc, attr: &attr, flags, eidx, evtdata); |
520 | if (ret) |
521 | return ret; |
522 | } |
523 | |
524 | set_bit(nr: ctr_idx, addr: kvpmu->pmc_in_use); |
525 | pmc->event_idx = eidx; |
526 | retdata->out_val = ctr_idx; |
527 | out: |
528 | retdata->err_val = sbiret; |
529 | |
530 | return 0; |
531 | } |
532 | |
533 | int kvm_riscv_vcpu_pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, |
534 | struct kvm_vcpu_sbi_return *retdata) |
535 | { |
536 | int ret; |
537 | |
538 | ret = pmu_ctr_read(vcpu, cidx, out_val: &retdata->out_val); |
539 | if (ret == -EINVAL) |
540 | retdata->err_val = SBI_ERR_INVALID_PARAM; |
541 | |
542 | return 0; |
543 | } |
544 | |
545 | void kvm_riscv_vcpu_pmu_init(struct kvm_vcpu *vcpu) |
546 | { |
547 | int i = 0, ret, num_hw_ctrs = 0, hpm_width = 0; |
548 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
549 | struct kvm_pmc *pmc; |
550 | |
551 | /* |
552 | * PMU functionality should be only available to guests if privilege mode |
553 | * filtering is available in the host. Otherwise, guest will always count |
554 | * events while the execution is in hypervisor mode. |
555 | */ |
556 | if (!riscv_isa_extension_available(NULL, SSCOFPMF)) |
557 | return; |
558 | |
559 | ret = riscv_pmu_get_hpm_info(&hpm_width, &num_hw_ctrs); |
560 | if (ret < 0 || !hpm_width || !num_hw_ctrs) |
561 | return; |
562 | |
563 | /* |
564 | * Increase the number of hardware counters to offset the time counter. |
565 | */ |
566 | kvpmu->num_hw_ctrs = num_hw_ctrs + 1; |
567 | kvpmu->num_fw_ctrs = SBI_PMU_FW_MAX; |
568 | memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event)); |
569 | |
570 | if (kvpmu->num_hw_ctrs > RISCV_KVM_MAX_HW_CTRS) { |
571 | pr_warn_once("Limiting the hardware counters to 32 as specified by the ISA" ); |
572 | kvpmu->num_hw_ctrs = RISCV_KVM_MAX_HW_CTRS; |
573 | } |
574 | |
575 | /* |
576 | * There is no correlation between the logical hardware counter and virtual counters. |
577 | * However, we need to encode a hpmcounter CSR in the counter info field so that |
578 | * KVM can trap n emulate the read. This works well in the migration use case as |
579 | * KVM doesn't care if the actual hpmcounter is available in the hardware or not. |
580 | */ |
581 | for (i = 0; i < kvm_pmu_num_counters(kvpmu); i++) { |
582 | /* TIME CSR shouldn't be read from perf interface */ |
583 | if (i == 1) |
584 | continue; |
585 | pmc = &kvpmu->pmc[i]; |
586 | pmc->idx = i; |
587 | pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID; |
588 | if (i < kvpmu->num_hw_ctrs) { |
589 | pmc->cinfo.type = SBI_PMU_CTR_TYPE_HW; |
590 | if (i < 3) |
591 | /* CY, IR counters */ |
592 | pmc->cinfo.width = 63; |
593 | else |
594 | pmc->cinfo.width = hpm_width; |
595 | /* |
596 | * The CSR number doesn't have any relation with the logical |
597 | * hardware counters. The CSR numbers are encoded sequentially |
598 | * to avoid maintaining a map between the virtual counter |
599 | * and CSR number. |
600 | */ |
601 | pmc->cinfo.csr = CSR_CYCLE + i; |
602 | } else { |
603 | pmc->cinfo.type = SBI_PMU_CTR_TYPE_FW; |
604 | pmc->cinfo.width = BITS_PER_LONG - 1; |
605 | } |
606 | } |
607 | |
608 | kvpmu->init_done = true; |
609 | } |
610 | |
611 | void kvm_riscv_vcpu_pmu_deinit(struct kvm_vcpu *vcpu) |
612 | { |
613 | struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); |
614 | struct kvm_pmc *pmc; |
615 | int i; |
616 | |
617 | if (!kvpmu) |
618 | return; |
619 | |
620 | for_each_set_bit(i, kvpmu->pmc_in_use, RISCV_MAX_COUNTERS) { |
621 | pmc = &kvpmu->pmc[i]; |
622 | pmc->counter_val = 0; |
623 | kvm_pmu_release_perf_event(pmc); |
624 | pmc->event_idx = SBI_PMU_EVENT_IDX_INVALID; |
625 | } |
626 | bitmap_zero(kvpmu->pmc_in_use, RISCV_MAX_COUNTERS); |
627 | memset(&kvpmu->fw_event, 0, SBI_PMU_FW_MAX * sizeof(struct kvm_fw_event)); |
628 | } |
629 | |
630 | void kvm_riscv_vcpu_pmu_reset(struct kvm_vcpu *vcpu) |
631 | { |
632 | kvm_riscv_vcpu_pmu_deinit(vcpu); |
633 | } |
634 | |