1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <errno.h> |
3 | #include <unistd.h> |
4 | #include <sys/syscall.h> |
5 | #include <perf/evsel.h> |
6 | #include <perf/cpumap.h> |
7 | #include <perf/threadmap.h> |
8 | #include <linux/list.h> |
9 | #include <internal/evsel.h> |
10 | #include <linux/zalloc.h> |
11 | #include <stdlib.h> |
12 | #include <internal/xyarray.h> |
13 | #include <internal/cpumap.h> |
14 | #include <internal/mmap.h> |
15 | #include <internal/threadmap.h> |
16 | #include <internal/lib.h> |
17 | #include <linux/string.h> |
18 | #include <sys/ioctl.h> |
19 | #include <sys/mman.h> |
20 | #include <asm/bug.h> |
21 | |
22 | void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr, |
23 | int idx) |
24 | { |
25 | INIT_LIST_HEAD(list: &evsel->node); |
26 | evsel->attr = *attr; |
27 | evsel->idx = idx; |
28 | evsel->leader = evsel; |
29 | } |
30 | |
31 | struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr) |
32 | { |
33 | struct perf_evsel *evsel = zalloc(sizeof(*evsel)); |
34 | |
35 | if (evsel != NULL) |
36 | perf_evsel__init(evsel, attr, idx: 0); |
37 | |
38 | return evsel; |
39 | } |
40 | |
41 | void perf_evsel__delete(struct perf_evsel *evsel) |
42 | { |
43 | free(evsel); |
44 | } |
45 | |
46 | #define FD(_evsel, _cpu_map_idx, _thread) \ |
47 | ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread)) |
48 | #define MMAP(_evsel, _cpu_map_idx, _thread) \ |
49 | (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \ |
50 | : NULL) |
51 | |
52 | int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads) |
53 | { |
54 | evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int)); |
55 | |
56 | if (evsel->fd) { |
57 | int idx, thread; |
58 | |
59 | for (idx = 0; idx < ncpus; idx++) { |
60 | for (thread = 0; thread < nthreads; thread++) { |
61 | int *fd = FD(evsel, idx, thread); |
62 | |
63 | if (fd) |
64 | *fd = -1; |
65 | } |
66 | } |
67 | } |
68 | |
69 | return evsel->fd != NULL ? 0 : -ENOMEM; |
70 | } |
71 | |
72 | static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads) |
73 | { |
74 | evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap)); |
75 | |
76 | return evsel->mmap != NULL ? 0 : -ENOMEM; |
77 | } |
78 | |
79 | static int |
80 | sys_perf_event_open(struct perf_event_attr *attr, |
81 | pid_t pid, struct perf_cpu cpu, int group_fd, |
82 | unsigned long flags) |
83 | { |
84 | return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags); |
85 | } |
86 | |
87 | static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd) |
88 | { |
89 | struct perf_evsel *leader = evsel->leader; |
90 | int *fd; |
91 | |
92 | if (evsel == leader) { |
93 | *group_fd = -1; |
94 | return 0; |
95 | } |
96 | |
97 | /* |
98 | * Leader must be already processed/open, |
99 | * if not it's a bug. |
100 | */ |
101 | if (!leader->fd) |
102 | return -ENOTCONN; |
103 | |
104 | fd = FD(leader, cpu_map_idx, thread); |
105 | if (fd == NULL || *fd == -1) |
106 | return -EBADF; |
107 | |
108 | *group_fd = *fd; |
109 | |
110 | return 0; |
111 | } |
112 | |
113 | int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus, |
114 | struct perf_thread_map *threads) |
115 | { |
116 | struct perf_cpu cpu; |
117 | int idx, thread, err = 0; |
118 | |
119 | if (cpus == NULL) { |
120 | static struct perf_cpu_map *empty_cpu_map; |
121 | |
122 | if (empty_cpu_map == NULL) { |
123 | empty_cpu_map = perf_cpu_map__new_any_cpu(); |
124 | if (empty_cpu_map == NULL) |
125 | return -ENOMEM; |
126 | } |
127 | |
128 | cpus = empty_cpu_map; |
129 | } |
130 | |
131 | if (threads == NULL) { |
132 | static struct perf_thread_map *empty_thread_map; |
133 | |
134 | if (empty_thread_map == NULL) { |
135 | empty_thread_map = perf_thread_map__new_dummy(); |
136 | if (empty_thread_map == NULL) |
137 | return -ENOMEM; |
138 | } |
139 | |
140 | threads = empty_thread_map; |
141 | } |
142 | |
143 | if (evsel->fd == NULL && |
144 | perf_evsel__alloc_fd(evsel, ncpus: perf_cpu_map__nr(cpus), nthreads: threads->nr) < 0) |
145 | return -ENOMEM; |
146 | |
147 | perf_cpu_map__for_each_cpu(cpu, idx, cpus) { |
148 | for (thread = 0; thread < threads->nr; thread++) { |
149 | int fd, group_fd, *evsel_fd; |
150 | |
151 | evsel_fd = FD(evsel, idx, thread); |
152 | if (evsel_fd == NULL) { |
153 | err = -EINVAL; |
154 | goto out; |
155 | } |
156 | |
157 | err = get_group_fd(evsel, cpu_map_idx: idx, thread, group_fd: &group_fd); |
158 | if (err < 0) |
159 | goto out; |
160 | |
161 | fd = sys_perf_event_open(attr: &evsel->attr, |
162 | pid: threads->map[thread].pid, |
163 | cpu: cpu, group_fd, flags: 0); |
164 | |
165 | if (fd < 0) { |
166 | err = -errno; |
167 | goto out; |
168 | } |
169 | |
170 | *evsel_fd = fd; |
171 | } |
172 | } |
173 | out: |
174 | if (err) |
175 | perf_evsel__close(evsel); |
176 | |
177 | return err; |
178 | } |
179 | |
180 | static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx) |
181 | { |
182 | int thread; |
183 | |
184 | for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) { |
185 | int *fd = FD(evsel, cpu_map_idx, thread); |
186 | |
187 | if (fd && *fd >= 0) { |
188 | close(*fd); |
189 | *fd = -1; |
190 | } |
191 | } |
192 | } |
193 | |
194 | void perf_evsel__close_fd(struct perf_evsel *evsel) |
195 | { |
196 | for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++) |
197 | perf_evsel__close_fd_cpu(evsel, cpu_map_idx: idx); |
198 | } |
199 | |
200 | void perf_evsel__free_fd(struct perf_evsel *evsel) |
201 | { |
202 | xyarray__delete(evsel->fd); |
203 | evsel->fd = NULL; |
204 | } |
205 | |
206 | void perf_evsel__close(struct perf_evsel *evsel) |
207 | { |
208 | if (evsel->fd == NULL) |
209 | return; |
210 | |
211 | perf_evsel__close_fd(evsel); |
212 | perf_evsel__free_fd(evsel); |
213 | } |
214 | |
215 | void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx) |
216 | { |
217 | if (evsel->fd == NULL) |
218 | return; |
219 | |
220 | perf_evsel__close_fd_cpu(evsel, cpu_map_idx); |
221 | } |
222 | |
223 | void perf_evsel__munmap(struct perf_evsel *evsel) |
224 | { |
225 | int idx, thread; |
226 | |
227 | if (evsel->fd == NULL || evsel->mmap == NULL) |
228 | return; |
229 | |
230 | for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { |
231 | for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { |
232 | int *fd = FD(evsel, idx, thread); |
233 | |
234 | if (fd == NULL || *fd < 0) |
235 | continue; |
236 | |
237 | perf_mmap__munmap(MMAP(evsel, idx, thread)); |
238 | } |
239 | } |
240 | |
241 | xyarray__delete(evsel->mmap); |
242 | evsel->mmap = NULL; |
243 | } |
244 | |
245 | int perf_evsel__mmap(struct perf_evsel *evsel, int pages) |
246 | { |
247 | int ret, idx, thread; |
248 | struct perf_mmap_param mp = { |
249 | .prot = PROT_READ | PROT_WRITE, |
250 | .mask = (pages * page_size) - 1, |
251 | }; |
252 | |
253 | if (evsel->fd == NULL || evsel->mmap) |
254 | return -EINVAL; |
255 | |
256 | if (perf_evsel__alloc_mmap(evsel, ncpus: xyarray__max_x(evsel->fd), nthreads: xyarray__max_y(evsel->fd)) < 0) |
257 | return -ENOMEM; |
258 | |
259 | for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { |
260 | for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { |
261 | int *fd = FD(evsel, idx, thread); |
262 | struct perf_mmap *map; |
263 | struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx); |
264 | |
265 | if (fd == NULL || *fd < 0) |
266 | continue; |
267 | |
268 | map = MMAP(evsel, idx, thread); |
269 | perf_mmap__init(map, NULL, false, NULL); |
270 | |
271 | ret = perf_mmap__mmap(map, &mp, *fd, cpu); |
272 | if (ret) { |
273 | perf_evsel__munmap(evsel); |
274 | return ret; |
275 | } |
276 | } |
277 | } |
278 | |
279 | return 0; |
280 | } |
281 | |
282 | void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread) |
283 | { |
284 | int *fd = FD(evsel, cpu_map_idx, thread); |
285 | |
286 | if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL) |
287 | return NULL; |
288 | |
289 | return MMAP(evsel, cpu_map_idx, thread)->base; |
290 | } |
291 | |
292 | int perf_evsel__read_size(struct perf_evsel *evsel) |
293 | { |
294 | u64 read_format = evsel->attr.read_format; |
295 | int entry = sizeof(u64); /* value */ |
296 | int size = 0; |
297 | int nr = 1; |
298 | |
299 | if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) |
300 | size += sizeof(u64); |
301 | |
302 | if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) |
303 | size += sizeof(u64); |
304 | |
305 | if (read_format & PERF_FORMAT_ID) |
306 | entry += sizeof(u64); |
307 | |
308 | if (read_format & PERF_FORMAT_LOST) |
309 | entry += sizeof(u64); |
310 | |
311 | if (read_format & PERF_FORMAT_GROUP) { |
312 | nr = evsel->nr_members; |
313 | size += sizeof(u64); |
314 | } |
315 | |
316 | size += entry * nr; |
317 | return size; |
318 | } |
319 | |
320 | /* This only reads values for the leader */ |
321 | static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx, |
322 | int thread, struct perf_counts_values *count) |
323 | { |
324 | size_t size = perf_evsel__read_size(evsel); |
325 | int *fd = FD(evsel, cpu_map_idx, thread); |
326 | u64 read_format = evsel->attr.read_format; |
327 | u64 *data; |
328 | int idx = 1; |
329 | |
330 | if (fd == NULL || *fd < 0) |
331 | return -EINVAL; |
332 | |
333 | data = calloc(1, size); |
334 | if (data == NULL) |
335 | return -ENOMEM; |
336 | |
337 | if (readn(*fd, data, size) <= 0) { |
338 | free(data); |
339 | return -errno; |
340 | } |
341 | |
342 | /* |
343 | * This reads only the leader event intentionally since we don't have |
344 | * perf counts values for sibling events. |
345 | */ |
346 | if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) |
347 | count->ena = data[idx++]; |
348 | if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) |
349 | count->run = data[idx++]; |
350 | |
351 | /* value is always available */ |
352 | count->val = data[idx++]; |
353 | if (read_format & PERF_FORMAT_ID) |
354 | count->id = data[idx++]; |
355 | if (read_format & PERF_FORMAT_LOST) |
356 | count->lost = data[idx++]; |
357 | |
358 | free(data); |
359 | return 0; |
360 | } |
361 | |
362 | /* |
363 | * The perf read format is very flexible. It needs to set the proper |
364 | * values according to the read format. |
365 | */ |
366 | static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf, |
367 | struct perf_counts_values *count) |
368 | { |
369 | u64 read_format = evsel->attr.read_format; |
370 | int n = 0; |
371 | |
372 | count->val = buf[n++]; |
373 | |
374 | if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) |
375 | count->ena = buf[n++]; |
376 | |
377 | if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) |
378 | count->run = buf[n++]; |
379 | |
380 | if (read_format & PERF_FORMAT_ID) |
381 | count->id = buf[n++]; |
382 | |
383 | if (read_format & PERF_FORMAT_LOST) |
384 | count->lost = buf[n++]; |
385 | } |
386 | |
387 | int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread, |
388 | struct perf_counts_values *count) |
389 | { |
390 | size_t size = perf_evsel__read_size(evsel); |
391 | int *fd = FD(evsel, cpu_map_idx, thread); |
392 | u64 read_format = evsel->attr.read_format; |
393 | struct perf_counts_values buf; |
394 | |
395 | memset(count, 0, sizeof(*count)); |
396 | |
397 | if (fd == NULL || *fd < 0) |
398 | return -EINVAL; |
399 | |
400 | if (read_format & PERF_FORMAT_GROUP) |
401 | return perf_evsel__read_group(evsel, cpu_map_idx, thread, count); |
402 | |
403 | if (MMAP(evsel, cpu_map_idx, thread) && |
404 | !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) && |
405 | !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count)) |
406 | return 0; |
407 | |
408 | if (readn(*fd, buf.values, size) <= 0) |
409 | return -errno; |
410 | |
411 | perf_evsel__adjust_values(evsel, buf: buf.values, count); |
412 | return 0; |
413 | } |
414 | |
415 | static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg, |
416 | int cpu_map_idx, int thread) |
417 | { |
418 | int *fd = FD(evsel, cpu_map_idx, thread); |
419 | |
420 | if (fd == NULL || *fd < 0) |
421 | return -1; |
422 | |
423 | return ioctl(*fd, ioc, arg); |
424 | } |
425 | |
426 | static int perf_evsel__run_ioctl(struct perf_evsel *evsel, |
427 | int ioc, void *arg, |
428 | int cpu_map_idx) |
429 | { |
430 | int thread; |
431 | |
432 | for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { |
433 | int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread); |
434 | |
435 | if (err) |
436 | return err; |
437 | } |
438 | |
439 | return 0; |
440 | } |
441 | |
442 | int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx) |
443 | { |
444 | return perf_evsel__run_ioctl(evsel, ioc: PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx); |
445 | } |
446 | |
447 | int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread) |
448 | { |
449 | struct perf_cpu cpu __maybe_unused; |
450 | int idx; |
451 | int err; |
452 | |
453 | perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) { |
454 | err = perf_evsel__ioctl(evsel, ioc: PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx: idx, thread); |
455 | if (err) |
456 | return err; |
457 | } |
458 | |
459 | return 0; |
460 | } |
461 | |
462 | int perf_evsel__enable(struct perf_evsel *evsel) |
463 | { |
464 | int i; |
465 | int err = 0; |
466 | |
467 | for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) |
468 | err = perf_evsel__run_ioctl(evsel, ioc: PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx: i); |
469 | return err; |
470 | } |
471 | |
472 | int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx) |
473 | { |
474 | return perf_evsel__run_ioctl(evsel, ioc: PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx); |
475 | } |
476 | |
477 | int perf_evsel__disable(struct perf_evsel *evsel) |
478 | { |
479 | int i; |
480 | int err = 0; |
481 | |
482 | for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) |
483 | err = perf_evsel__run_ioctl(evsel, ioc: PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx: i); |
484 | return err; |
485 | } |
486 | |
487 | int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter) |
488 | { |
489 | int err = 0, i; |
490 | |
491 | for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++) |
492 | err = perf_evsel__run_ioctl(evsel, |
493 | ioc: PERF_EVENT_IOC_SET_FILTER, |
494 | arg: (void *)filter, cpu_map_idx: i); |
495 | return err; |
496 | } |
497 | |
498 | struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel) |
499 | { |
500 | return evsel->cpus; |
501 | } |
502 | |
503 | struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel) |
504 | { |
505 | return evsel->threads; |
506 | } |
507 | |
508 | struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel) |
509 | { |
510 | return &evsel->attr; |
511 | } |
512 | |
513 | int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads) |
514 | { |
515 | if (ncpus == 0 || nthreads == 0) |
516 | return 0; |
517 | |
518 | evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id)); |
519 | if (evsel->sample_id == NULL) |
520 | return -ENOMEM; |
521 | |
522 | evsel->id = zalloc(ncpus * nthreads * sizeof(u64)); |
523 | if (evsel->id == NULL) { |
524 | xyarray__delete(evsel->sample_id); |
525 | evsel->sample_id = NULL; |
526 | return -ENOMEM; |
527 | } |
528 | |
529 | return 0; |
530 | } |
531 | |
532 | void perf_evsel__free_id(struct perf_evsel *evsel) |
533 | { |
534 | xyarray__delete(evsel->sample_id); |
535 | evsel->sample_id = NULL; |
536 | zfree(&evsel->id); |
537 | evsel->ids = 0; |
538 | } |
539 | |
540 | void perf_counts_values__scale(struct perf_counts_values *count, |
541 | bool scale, __s8 *pscaled) |
542 | { |
543 | s8 scaled = 0; |
544 | |
545 | if (scale) { |
546 | if (count->run == 0) { |
547 | scaled = -1; |
548 | count->val = 0; |
549 | } else if (count->run < count->ena) { |
550 | scaled = 1; |
551 | count->val = (u64)((double)count->val * count->ena / count->run); |
552 | } |
553 | } |
554 | |
555 | if (pscaled) |
556 | *pscaled = scaled; |
557 | } |
558 | |