1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <sys/mman.h> |
3 | #include <inttypes.h> |
4 | #include <asm/bug.h> |
5 | #include <errno.h> |
6 | #include <string.h> |
7 | #include <linux/ring_buffer.h> |
8 | #include <linux/perf_event.h> |
9 | #include <perf/mmap.h> |
10 | #include <perf/event.h> |
11 | #include <perf/evsel.h> |
12 | #include <internal/mmap.h> |
13 | #include <internal/lib.h> |
14 | #include <linux/kernel.h> |
15 | #include <linux/math64.h> |
16 | #include <linux/stringify.h> |
17 | #include "internal.h" |
18 | |
19 | void perf_mmap__init(struct perf_mmap *map, struct perf_mmap *prev, |
20 | bool overwrite, libperf_unmap_cb_t unmap_cb) |
21 | { |
22 | /* Assume fields were zero initialized. */ |
23 | map->fd = -1; |
24 | map->overwrite = overwrite; |
25 | map->unmap_cb = unmap_cb; |
26 | refcount_set(r: &map->refcnt, n: 0); |
27 | if (prev) |
28 | prev->next = map; |
29 | } |
30 | |
31 | size_t perf_mmap__mmap_len(struct perf_mmap *map) |
32 | { |
33 | return map->mask + 1 + page_size; |
34 | } |
35 | |
36 | int perf_mmap__mmap(struct perf_mmap *map, struct perf_mmap_param *mp, |
37 | int fd, struct perf_cpu cpu) |
38 | { |
39 | map->prev = 0; |
40 | map->mask = mp->mask; |
41 | map->base = mmap(NULL, perf_mmap__mmap_len(map), mp->prot, |
42 | MAP_SHARED, fd, 0); |
43 | if (map->base == MAP_FAILED) { |
44 | map->base = NULL; |
45 | return -1; |
46 | } |
47 | |
48 | map->fd = fd; |
49 | map->cpu = cpu; |
50 | return 0; |
51 | } |
52 | |
53 | void perf_mmap__munmap(struct perf_mmap *map) |
54 | { |
55 | if (!map) |
56 | return; |
57 | |
58 | zfree(&map->event_copy); |
59 | map->event_copy_sz = 0; |
60 | if (map->base) { |
61 | munmap(map->base, perf_mmap__mmap_len(map)); |
62 | map->base = NULL; |
63 | map->fd = -1; |
64 | refcount_set(r: &map->refcnt, n: 0); |
65 | } |
66 | if (map->unmap_cb) |
67 | map->unmap_cb(map); |
68 | } |
69 | |
70 | void perf_mmap__get(struct perf_mmap *map) |
71 | { |
72 | refcount_inc(r: &map->refcnt); |
73 | } |
74 | |
75 | void perf_mmap__put(struct perf_mmap *map) |
76 | { |
77 | BUG_ON(map->base && refcount_read(&map->refcnt) == 0); |
78 | |
79 | if (refcount_dec_and_test(r: &map->refcnt)) |
80 | perf_mmap__munmap(map); |
81 | } |
82 | |
83 | static inline void perf_mmap__write_tail(struct perf_mmap *md, u64 tail) |
84 | { |
85 | ring_buffer_write_tail(md->base, tail); |
86 | } |
87 | |
88 | u64 perf_mmap__read_head(struct perf_mmap *map) |
89 | { |
90 | return ring_buffer_read_head(map->base); |
91 | } |
92 | |
93 | static bool perf_mmap__empty(struct perf_mmap *map) |
94 | { |
95 | struct perf_event_mmap_page *pc = map->base; |
96 | |
97 | return perf_mmap__read_head(map) == map->prev && !pc->aux_size; |
98 | } |
99 | |
100 | void perf_mmap__consume(struct perf_mmap *map) |
101 | { |
102 | if (!map->overwrite) { |
103 | u64 old = map->prev; |
104 | |
105 | perf_mmap__write_tail(md: map, tail: old); |
106 | } |
107 | |
108 | if (refcount_read(r: &map->refcnt) == 1 && perf_mmap__empty(map)) |
109 | perf_mmap__put(map); |
110 | } |
111 | |
112 | static int overwrite_rb_find_range(void *buf, int mask, u64 *start, u64 *end) |
113 | { |
114 | struct perf_event_header *; |
115 | u64 evt_head = *start; |
116 | int size = mask + 1; |
117 | |
118 | pr_debug2("%s: buf=%p, start=%" PRIx64"\n" , __func__, buf, *start); |
119 | pheader = (struct perf_event_header *)(buf + (*start & mask)); |
120 | while (true) { |
121 | if (evt_head - *start >= (unsigned int)size) { |
122 | pr_debug("Finished reading overwrite ring buffer: rewind\n" ); |
123 | if (evt_head - *start > (unsigned int)size) |
124 | evt_head -= pheader->size; |
125 | *end = evt_head; |
126 | return 0; |
127 | } |
128 | |
129 | pheader = (struct perf_event_header *)(buf + (evt_head & mask)); |
130 | |
131 | if (pheader->size == 0) { |
132 | pr_debug("Finished reading overwrite ring buffer: get start\n" ); |
133 | *end = evt_head; |
134 | return 0; |
135 | } |
136 | |
137 | evt_head += pheader->size; |
138 | pr_debug3("move evt_head: %" PRIx64"\n" , evt_head); |
139 | } |
140 | WARN_ONCE(1, "Shouldn't get here\n" ); |
141 | return -1; |
142 | } |
143 | |
144 | /* |
145 | * Report the start and end of the available data in ringbuffer |
146 | */ |
147 | static int __perf_mmap__read_init(struct perf_mmap *md) |
148 | { |
149 | u64 head = perf_mmap__read_head(map: md); |
150 | u64 old = md->prev; |
151 | unsigned char *data = md->base + page_size; |
152 | unsigned long size; |
153 | |
154 | md->start = md->overwrite ? head : old; |
155 | md->end = md->overwrite ? old : head; |
156 | |
157 | if ((md->end - md->start) < md->flush) |
158 | return -EAGAIN; |
159 | |
160 | size = md->end - md->start; |
161 | if (size > (unsigned long)(md->mask) + 1) { |
162 | if (!md->overwrite) { |
163 | WARN_ONCE(1, "failed to keep up with mmap data. (warn only once)\n" ); |
164 | |
165 | md->prev = head; |
166 | perf_mmap__consume(map: md); |
167 | return -EAGAIN; |
168 | } |
169 | |
170 | /* |
171 | * Backward ring buffer is full. We still have a chance to read |
172 | * most of data from it. |
173 | */ |
174 | if (overwrite_rb_find_range(buf: data, mask: md->mask, start: &md->start, end: &md->end)) |
175 | return -EINVAL; |
176 | } |
177 | |
178 | return 0; |
179 | } |
180 | |
181 | int perf_mmap__read_init(struct perf_mmap *map) |
182 | { |
183 | /* |
184 | * Check if event was unmapped due to a POLLHUP/POLLERR. |
185 | */ |
186 | if (!refcount_read(r: &map->refcnt)) |
187 | return -ENOENT; |
188 | |
189 | return __perf_mmap__read_init(md: map); |
190 | } |
191 | |
192 | /* |
193 | * Mandatory for overwrite mode |
194 | * The direction of overwrite mode is backward. |
195 | * The last perf_mmap__read() will set tail to map->core.prev. |
196 | * Need to correct the map->core.prev to head which is the end of next read. |
197 | */ |
198 | void perf_mmap__read_done(struct perf_mmap *map) |
199 | { |
200 | /* |
201 | * Check if event was unmapped due to a POLLHUP/POLLERR. |
202 | */ |
203 | if (!refcount_read(r: &map->refcnt)) |
204 | return; |
205 | |
206 | map->prev = perf_mmap__read_head(map); |
207 | } |
208 | |
209 | /* When check_messup is true, 'end' must points to a good entry */ |
210 | static union perf_event *perf_mmap__read(struct perf_mmap *map, |
211 | u64 *startp, u64 end) |
212 | { |
213 | unsigned char *data = map->base + page_size; |
214 | union perf_event *event = NULL; |
215 | int diff = end - *startp; |
216 | |
217 | if (diff >= (int)sizeof(event->header)) { |
218 | size_t size; |
219 | |
220 | event = (union perf_event *)&data[*startp & map->mask]; |
221 | size = event->header.size; |
222 | |
223 | if (size < sizeof(event->header) || diff < (int)size) |
224 | return NULL; |
225 | |
226 | /* |
227 | * Event straddles the mmap boundary -- header should always |
228 | * be inside due to u64 alignment of output. |
229 | */ |
230 | if ((*startp & map->mask) + size != ((*startp + size) & map->mask)) { |
231 | unsigned int offset = *startp; |
232 | unsigned int len = size, cpy; |
233 | void *dst = map->event_copy; |
234 | |
235 | if (size > map->event_copy_sz) { |
236 | dst = realloc(map->event_copy, size); |
237 | if (!dst) |
238 | return NULL; |
239 | map->event_copy = dst; |
240 | map->event_copy_sz = size; |
241 | } |
242 | |
243 | do { |
244 | cpy = min(map->mask + 1 - (offset & map->mask), len); |
245 | memcpy(dst, &data[offset & map->mask], cpy); |
246 | offset += cpy; |
247 | dst += cpy; |
248 | len -= cpy; |
249 | } while (len); |
250 | |
251 | event = (union perf_event *)map->event_copy; |
252 | } |
253 | |
254 | *startp += size; |
255 | } |
256 | |
257 | return event; |
258 | } |
259 | |
260 | /* |
261 | * Read event from ring buffer one by one. |
262 | * Return one event for each call. |
263 | * |
264 | * Usage: |
265 | * perf_mmap__read_init() |
266 | * while(event = perf_mmap__read_event()) { |
267 | * //process the event |
268 | * perf_mmap__consume() |
269 | * } |
270 | * perf_mmap__read_done() |
271 | */ |
272 | union perf_event *perf_mmap__read_event(struct perf_mmap *map) |
273 | { |
274 | union perf_event *event; |
275 | |
276 | /* |
277 | * Check if event was unmapped due to a POLLHUP/POLLERR. |
278 | */ |
279 | if (!refcount_read(r: &map->refcnt)) |
280 | return NULL; |
281 | |
282 | /* non-overwirte doesn't pause the ringbuffer */ |
283 | if (!map->overwrite) |
284 | map->end = perf_mmap__read_head(map); |
285 | |
286 | event = perf_mmap__read(map, startp: &map->start, end: map->end); |
287 | |
288 | if (!map->overwrite) |
289 | map->prev = map->start; |
290 | |
291 | return event; |
292 | } |
293 | |
294 | #if defined(__i386__) || defined(__x86_64__) |
295 | static u64 read_perf_counter(unsigned int counter) |
296 | { |
297 | unsigned int low, high; |
298 | |
299 | asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter)); |
300 | |
301 | return low | ((u64)high) << 32; |
302 | } |
303 | |
304 | static u64 read_timestamp(void) |
305 | { |
306 | unsigned int low, high; |
307 | |
308 | asm volatile("rdtsc" : "=a" (low), "=d" (high)); |
309 | |
310 | return low | ((u64)high) << 32; |
311 | } |
312 | #elif defined(__aarch64__) |
313 | #define read_sysreg(r) ({ \ |
314 | u64 __val; \ |
315 | asm volatile("mrs %0, " __stringify(r) : "=r" (__val)); \ |
316 | __val; \ |
317 | }) |
318 | |
319 | static u64 read_pmccntr(void) |
320 | { |
321 | return read_sysreg(pmccntr_el0); |
322 | } |
323 | |
324 | #define PMEVCNTR_READ(idx) \ |
325 | static u64 read_pmevcntr_##idx(void) { \ |
326 | return read_sysreg(pmevcntr##idx##_el0); \ |
327 | } |
328 | |
329 | PMEVCNTR_READ(0); |
330 | PMEVCNTR_READ(1); |
331 | PMEVCNTR_READ(2); |
332 | PMEVCNTR_READ(3); |
333 | PMEVCNTR_READ(4); |
334 | PMEVCNTR_READ(5); |
335 | PMEVCNTR_READ(6); |
336 | PMEVCNTR_READ(7); |
337 | PMEVCNTR_READ(8); |
338 | PMEVCNTR_READ(9); |
339 | PMEVCNTR_READ(10); |
340 | PMEVCNTR_READ(11); |
341 | PMEVCNTR_READ(12); |
342 | PMEVCNTR_READ(13); |
343 | PMEVCNTR_READ(14); |
344 | PMEVCNTR_READ(15); |
345 | PMEVCNTR_READ(16); |
346 | PMEVCNTR_READ(17); |
347 | PMEVCNTR_READ(18); |
348 | PMEVCNTR_READ(19); |
349 | PMEVCNTR_READ(20); |
350 | PMEVCNTR_READ(21); |
351 | PMEVCNTR_READ(22); |
352 | PMEVCNTR_READ(23); |
353 | PMEVCNTR_READ(24); |
354 | PMEVCNTR_READ(25); |
355 | PMEVCNTR_READ(26); |
356 | PMEVCNTR_READ(27); |
357 | PMEVCNTR_READ(28); |
358 | PMEVCNTR_READ(29); |
359 | PMEVCNTR_READ(30); |
360 | |
361 | /* |
362 | * Read a value direct from PMEVCNTR<idx> |
363 | */ |
364 | static u64 read_perf_counter(unsigned int counter) |
365 | { |
366 | static u64 (* const read_f[])(void) = { |
367 | read_pmevcntr_0, |
368 | read_pmevcntr_1, |
369 | read_pmevcntr_2, |
370 | read_pmevcntr_3, |
371 | read_pmevcntr_4, |
372 | read_pmevcntr_5, |
373 | read_pmevcntr_6, |
374 | read_pmevcntr_7, |
375 | read_pmevcntr_8, |
376 | read_pmevcntr_9, |
377 | read_pmevcntr_10, |
378 | read_pmevcntr_11, |
379 | read_pmevcntr_13, |
380 | read_pmevcntr_12, |
381 | read_pmevcntr_14, |
382 | read_pmevcntr_15, |
383 | read_pmevcntr_16, |
384 | read_pmevcntr_17, |
385 | read_pmevcntr_18, |
386 | read_pmevcntr_19, |
387 | read_pmevcntr_20, |
388 | read_pmevcntr_21, |
389 | read_pmevcntr_22, |
390 | read_pmevcntr_23, |
391 | read_pmevcntr_24, |
392 | read_pmevcntr_25, |
393 | read_pmevcntr_26, |
394 | read_pmevcntr_27, |
395 | read_pmevcntr_28, |
396 | read_pmevcntr_29, |
397 | read_pmevcntr_30, |
398 | read_pmccntr |
399 | }; |
400 | |
401 | if (counter < ARRAY_SIZE(read_f)) |
402 | return (read_f[counter])(); |
403 | |
404 | return 0; |
405 | } |
406 | |
407 | static u64 read_timestamp(void) { return read_sysreg(cntvct_el0); } |
408 | |
409 | /* __riscv_xlen contains the witdh of the native base integer, here 64-bit */ |
410 | #elif defined(__riscv) && __riscv_xlen == 64 |
411 | |
412 | /* TODO: implement rv32 support */ |
413 | |
414 | #define CSR_CYCLE 0xc00 |
415 | #define CSR_TIME 0xc01 |
416 | |
417 | #define csr_read(csr) \ |
418 | ({ \ |
419 | register unsigned long __v; \ |
420 | __asm__ __volatile__ ("csrr %0, %1" \ |
421 | : "=r" (__v) \ |
422 | : "i" (csr) : ); \ |
423 | __v; \ |
424 | }) |
425 | |
426 | static unsigned long csr_read_num(int csr_num) |
427 | { |
428 | #define switchcase_csr_read(__csr_num, __val) {\ |
429 | case __csr_num: \ |
430 | __val = csr_read(__csr_num); \ |
431 | break; } |
432 | #define switchcase_csr_read_2(__csr_num, __val) {\ |
433 | switchcase_csr_read(__csr_num + 0, __val) \ |
434 | switchcase_csr_read(__csr_num + 1, __val)} |
435 | #define switchcase_csr_read_4(__csr_num, __val) {\ |
436 | switchcase_csr_read_2(__csr_num + 0, __val) \ |
437 | switchcase_csr_read_2(__csr_num + 2, __val)} |
438 | #define switchcase_csr_read_8(__csr_num, __val) {\ |
439 | switchcase_csr_read_4(__csr_num + 0, __val) \ |
440 | switchcase_csr_read_4(__csr_num + 4, __val)} |
441 | #define switchcase_csr_read_16(__csr_num, __val) {\ |
442 | switchcase_csr_read_8(__csr_num + 0, __val) \ |
443 | switchcase_csr_read_8(__csr_num + 8, __val)} |
444 | #define switchcase_csr_read_32(__csr_num, __val) {\ |
445 | switchcase_csr_read_16(__csr_num + 0, __val) \ |
446 | switchcase_csr_read_16(__csr_num + 16, __val)} |
447 | |
448 | unsigned long ret = 0; |
449 | |
450 | switch (csr_num) { |
451 | switchcase_csr_read_32(CSR_CYCLE, ret) |
452 | default: |
453 | break; |
454 | } |
455 | |
456 | return ret; |
457 | #undef switchcase_csr_read_32 |
458 | #undef switchcase_csr_read_16 |
459 | #undef switchcase_csr_read_8 |
460 | #undef switchcase_csr_read_4 |
461 | #undef switchcase_csr_read_2 |
462 | #undef switchcase_csr_read |
463 | } |
464 | |
465 | static u64 read_perf_counter(unsigned int counter) |
466 | { |
467 | return csr_read_num(CSR_CYCLE + counter); |
468 | } |
469 | |
470 | static u64 read_timestamp(void) |
471 | { |
472 | return csr_read_num(CSR_TIME); |
473 | } |
474 | |
475 | #else |
476 | static u64 read_perf_counter(unsigned int counter __maybe_unused) { return 0; } |
477 | static u64 read_timestamp(void) { return 0; } |
478 | #endif |
479 | |
480 | int perf_mmap__read_self(struct perf_mmap *map, struct perf_counts_values *count) |
481 | { |
482 | struct perf_event_mmap_page *pc = map->base; |
483 | u32 seq, idx, time_mult = 0, time_shift = 0; |
484 | u64 cnt, cyc = 0, time_offset = 0, time_cycles = 0, time_mask = ~0ULL; |
485 | |
486 | if (!pc || !pc->cap_user_rdpmc) |
487 | return -1; |
488 | |
489 | do { |
490 | seq = READ_ONCE(pc->lock); |
491 | barrier(); |
492 | |
493 | count->ena = READ_ONCE(pc->time_enabled); |
494 | count->run = READ_ONCE(pc->time_running); |
495 | |
496 | if (pc->cap_user_time && count->ena != count->run) { |
497 | cyc = read_timestamp(); |
498 | time_mult = READ_ONCE(pc->time_mult); |
499 | time_shift = READ_ONCE(pc->time_shift); |
500 | time_offset = READ_ONCE(pc->time_offset); |
501 | |
502 | if (pc->cap_user_time_short) { |
503 | time_cycles = READ_ONCE(pc->time_cycles); |
504 | time_mask = READ_ONCE(pc->time_mask); |
505 | } |
506 | } |
507 | |
508 | idx = READ_ONCE(pc->index); |
509 | cnt = READ_ONCE(pc->offset); |
510 | if (pc->cap_user_rdpmc && idx) { |
511 | s64 evcnt = read_perf_counter(counter: idx - 1); |
512 | u16 width = READ_ONCE(pc->pmc_width); |
513 | |
514 | evcnt <<= 64 - width; |
515 | evcnt >>= 64 - width; |
516 | cnt += evcnt; |
517 | } else |
518 | return -1; |
519 | |
520 | barrier(); |
521 | } while (READ_ONCE(pc->lock) != seq); |
522 | |
523 | if (count->ena != count->run) { |
524 | u64 delta; |
525 | |
526 | /* Adjust for cap_usr_time_short, a nop if not */ |
527 | cyc = time_cycles + ((cyc - time_cycles) & time_mask); |
528 | |
529 | delta = time_offset + mul_u64_u32_shr(a: cyc, mul: time_mult, shift: time_shift); |
530 | |
531 | count->ena += delta; |
532 | if (idx) |
533 | count->run += delta; |
534 | } |
535 | |
536 | count->val = cnt; |
537 | |
538 | return 0; |
539 | } |
540 | |