1// SPDX-License-Identifier: GPL-2.0
2#include <errno.h>
3#include <unistd.h>
4#include <sys/syscall.h>
5#include <perf/evsel.h>
6#include <perf/cpumap.h>
7#include <perf/threadmap.h>
8#include <linux/hash.h>
9#include <linux/list.h>
10#include <internal/evsel.h>
11#include <linux/zalloc.h>
12#include <stdlib.h>
13#include <internal/xyarray.h>
14#include <internal/cpumap.h>
15#include <internal/mmap.h>
16#include <internal/threadmap.h>
17#include <internal/lib.h>
18#include <linux/string.h>
19#include <sys/ioctl.h>
20#include <sys/mman.h>
21#include <asm/bug.h>
22
23void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr,
24 int idx)
25{
26 INIT_LIST_HEAD(list: &evsel->node);
27 INIT_LIST_HEAD(list: &evsel->per_stream_periods);
28 evsel->attr = *attr;
29 evsel->idx = idx;
30 evsel->leader = evsel;
31}
32
33struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr)
34{
35 struct perf_evsel *evsel = zalloc(sizeof(*evsel));
36
37 if (evsel != NULL)
38 perf_evsel__init(evsel, attr, idx: 0);
39
40 return evsel;
41}
42
43void perf_evsel__exit(struct perf_evsel *evsel)
44{
45 assert(evsel->fd == NULL); /* If not fds were not closed. */
46 assert(evsel->mmap == NULL); /* If not munmap wasn't called. */
47 assert(evsel->sample_id == NULL); /* If not free_id wasn't called. */
48 perf_cpu_map__put(evsel->cpus);
49 perf_cpu_map__put(evsel->pmu_cpus);
50 perf_thread_map__put(evsel->threads);
51}
52
53void perf_evsel__delete(struct perf_evsel *evsel)
54{
55 perf_evsel__exit(evsel);
56 free(evsel);
57}
58
59#define FD(_evsel, _cpu_map_idx, _thread) \
60 ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread))
61#define MMAP(_evsel, _cpu_map_idx, _thread) \
62 (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \
63 : NULL)
64
65int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
66{
67 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
68
69 if (evsel->fd) {
70 int idx, thread;
71
72 for (idx = 0; idx < ncpus; idx++) {
73 for (thread = 0; thread < nthreads; thread++) {
74 int *fd = FD(evsel, idx, thread);
75
76 if (fd)
77 *fd = -1;
78 }
79 }
80 }
81
82 return evsel->fd != NULL ? 0 : -ENOMEM;
83}
84
85static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads)
86{
87 evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap));
88
89 return evsel->mmap != NULL ? 0 : -ENOMEM;
90}
91
92static int
93sys_perf_event_open(struct perf_event_attr *attr,
94 pid_t pid, struct perf_cpu cpu, int group_fd,
95 unsigned long flags)
96{
97 return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags);
98}
99
100static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd)
101{
102 struct perf_evsel *leader = evsel->leader;
103 int *fd;
104
105 if (evsel == leader) {
106 *group_fd = -1;
107 return 0;
108 }
109
110 /*
111 * Leader must be already processed/open,
112 * if not it's a bug.
113 */
114 if (!leader->fd)
115 return -ENOTCONN;
116
117 fd = FD(leader, cpu_map_idx, thread);
118 if (fd == NULL || *fd == -1)
119 return -EBADF;
120
121 *group_fd = *fd;
122
123 return 0;
124}
125
126int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus,
127 struct perf_thread_map *threads)
128{
129 struct perf_cpu cpu;
130 int idx, thread, err = 0;
131
132 if (cpus == NULL) {
133 static struct perf_cpu_map *empty_cpu_map;
134
135 if (empty_cpu_map == NULL) {
136 empty_cpu_map = perf_cpu_map__new_any_cpu();
137 if (empty_cpu_map == NULL)
138 return -ENOMEM;
139 }
140
141 cpus = empty_cpu_map;
142 }
143
144 if (threads == NULL) {
145 static struct perf_thread_map *empty_thread_map;
146
147 if (empty_thread_map == NULL) {
148 empty_thread_map = perf_thread_map__new_dummy();
149 if (empty_thread_map == NULL)
150 return -ENOMEM;
151 }
152
153 threads = empty_thread_map;
154 }
155
156 if (evsel->fd == NULL &&
157 perf_evsel__alloc_fd(evsel, ncpus: perf_cpu_map__nr(cpus), nthreads: threads->nr) < 0)
158 return -ENOMEM;
159
160 perf_cpu_map__for_each_cpu(cpu, idx, cpus) {
161 for (thread = 0; thread < threads->nr; thread++) {
162 int fd, group_fd, *evsel_fd;
163
164 evsel_fd = FD(evsel, idx, thread);
165 if (evsel_fd == NULL) {
166 err = -EINVAL;
167 goto out;
168 }
169
170 err = get_group_fd(evsel, cpu_map_idx: idx, thread, group_fd: &group_fd);
171 if (err < 0)
172 goto out;
173
174 fd = sys_perf_event_open(attr: &evsel->attr,
175 pid: threads->map[thread].pid,
176 cpu: cpu, group_fd, flags: 0);
177
178 if (fd < 0) {
179 err = -errno;
180 goto out;
181 }
182
183 *evsel_fd = fd;
184 }
185 }
186out:
187 if (err)
188 perf_evsel__close(evsel);
189
190 return err;
191}
192
193static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx)
194{
195 int thread;
196
197 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) {
198 int *fd = FD(evsel, cpu_map_idx, thread);
199
200 if (fd && *fd >= 0) {
201 close(*fd);
202 *fd = -1;
203 }
204 }
205}
206
207void perf_evsel__close_fd(struct perf_evsel *evsel)
208{
209 for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++)
210 perf_evsel__close_fd_cpu(evsel, cpu_map_idx: idx);
211}
212
213void perf_evsel__free_fd(struct perf_evsel *evsel)
214{
215 xyarray__delete(evsel->fd);
216 evsel->fd = NULL;
217}
218
219void perf_evsel__close(struct perf_evsel *evsel)
220{
221 if (evsel->fd == NULL)
222 return;
223
224 perf_evsel__close_fd(evsel);
225 perf_evsel__free_fd(evsel);
226}
227
228void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx)
229{
230 if (evsel->fd == NULL)
231 return;
232
233 perf_evsel__close_fd_cpu(evsel, cpu_map_idx);
234}
235
236void perf_evsel__munmap(struct perf_evsel *evsel)
237{
238 int idx, thread;
239
240 if (evsel->fd == NULL || evsel->mmap == NULL)
241 return;
242
243 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
244 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
245 int *fd = FD(evsel, idx, thread);
246
247 if (fd == NULL || *fd < 0)
248 continue;
249
250 perf_mmap__munmap(MMAP(evsel, idx, thread));
251 }
252 }
253
254 xyarray__delete(evsel->mmap);
255 evsel->mmap = NULL;
256}
257
258int perf_evsel__mmap(struct perf_evsel *evsel, int pages)
259{
260 int ret, idx, thread;
261 struct perf_mmap_param mp = {
262 .prot = PROT_READ | PROT_WRITE,
263 .mask = (pages * page_size) - 1,
264 };
265
266 if (evsel->fd == NULL || evsel->mmap)
267 return -EINVAL;
268
269 if (perf_evsel__alloc_mmap(evsel, ncpus: xyarray__max_x(evsel->fd), nthreads: xyarray__max_y(evsel->fd)) < 0)
270 return -ENOMEM;
271
272 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
273 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
274 int *fd = FD(evsel, idx, thread);
275 struct perf_mmap *map;
276 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx);
277
278 if (fd == NULL || *fd < 0)
279 continue;
280
281 map = MMAP(evsel, idx, thread);
282 perf_mmap__init(map, NULL, false, NULL);
283
284 ret = perf_mmap__mmap(map, &mp, *fd, cpu);
285 if (ret) {
286 perf_evsel__munmap(evsel);
287 return ret;
288 }
289 }
290 }
291
292 return 0;
293}
294
295void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread)
296{
297 int *fd = FD(evsel, cpu_map_idx, thread);
298
299 if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL)
300 return NULL;
301
302 return MMAP(evsel, cpu_map_idx, thread)->base;
303}
304
305int perf_evsel__read_size(struct perf_evsel *evsel)
306{
307 u64 read_format = evsel->attr.read_format;
308 int entry = sizeof(u64); /* value */
309 int size = 0;
310 int nr = 1;
311
312 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
313 size += sizeof(u64);
314
315 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
316 size += sizeof(u64);
317
318 if (read_format & PERF_FORMAT_ID)
319 entry += sizeof(u64);
320
321 if (read_format & PERF_FORMAT_LOST)
322 entry += sizeof(u64);
323
324 if (read_format & PERF_FORMAT_GROUP) {
325 nr = evsel->nr_members;
326 size += sizeof(u64);
327 }
328
329 size += entry * nr;
330 return size;
331}
332
333/* This only reads values for the leader */
334static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
335 int thread, struct perf_counts_values *count)
336{
337 size_t size = perf_evsel__read_size(evsel);
338 int *fd = FD(evsel, cpu_map_idx, thread);
339 u64 read_format = evsel->attr.read_format;
340 u64 *data;
341 int idx = 1;
342
343 if (fd == NULL || *fd < 0)
344 return -EINVAL;
345
346 data = calloc(1, size);
347 if (data == NULL)
348 return -ENOMEM;
349
350 if (readn(*fd, data, size) <= 0) {
351 free(data);
352 return -errno;
353 }
354
355 /*
356 * This reads only the leader event intentionally since we don't have
357 * perf counts values for sibling events.
358 */
359 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
360 count->ena = data[idx++];
361 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
362 count->run = data[idx++];
363
364 /* value is always available */
365 count->val = data[idx++];
366 if (read_format & PERF_FORMAT_ID)
367 count->id = data[idx++];
368 if (read_format & PERF_FORMAT_LOST)
369 count->lost = data[idx++];
370
371 free(data);
372 return 0;
373}
374
375/*
376 * The perf read format is very flexible. It needs to set the proper
377 * values according to the read format.
378 */
379static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf,
380 struct perf_counts_values *count)
381{
382 u64 read_format = evsel->attr.read_format;
383 int n = 0;
384
385 count->val = buf[n++];
386
387 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
388 count->ena = buf[n++];
389
390 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
391 count->run = buf[n++];
392
393 if (read_format & PERF_FORMAT_ID)
394 count->id = buf[n++];
395
396 if (read_format & PERF_FORMAT_LOST)
397 count->lost = buf[n++];
398}
399
400int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread,
401 struct perf_counts_values *count)
402{
403 size_t size = perf_evsel__read_size(evsel);
404 int *fd = FD(evsel, cpu_map_idx, thread);
405 u64 read_format = evsel->attr.read_format;
406 struct perf_counts_values buf;
407
408 memset(count, 0, sizeof(*count));
409
410 if (fd == NULL || *fd < 0)
411 return -EINVAL;
412
413 if (read_format & PERF_FORMAT_GROUP)
414 return perf_evsel__read_group(evsel, cpu_map_idx, thread, count);
415
416 if (MMAP(evsel, cpu_map_idx, thread) &&
417 !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) &&
418 !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count))
419 return 0;
420
421 if (readn(*fd, buf.values, size) <= 0)
422 return -errno;
423
424 perf_evsel__adjust_values(evsel, buf: buf.values, count);
425 return 0;
426}
427
428static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg,
429 int cpu_map_idx, int thread)
430{
431 int *fd = FD(evsel, cpu_map_idx, thread);
432
433 if (fd == NULL || *fd < 0)
434 return -1;
435
436 return ioctl(*fd, ioc, arg);
437}
438
439static int perf_evsel__run_ioctl(struct perf_evsel *evsel,
440 int ioc, void *arg,
441 int cpu_map_idx)
442{
443 int thread;
444
445 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
446 int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread);
447
448 if (err)
449 return err;
450 }
451
452 return 0;
453}
454
455int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
456{
457 return perf_evsel__run_ioctl(evsel, ioc: PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx);
458}
459
460int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread)
461{
462 struct perf_cpu cpu __maybe_unused;
463 int idx;
464 int err;
465
466 perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) {
467 err = perf_evsel__ioctl(evsel, ioc: PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx: idx, thread);
468 if (err)
469 return err;
470 }
471
472 return 0;
473}
474
475int perf_evsel__enable(struct perf_evsel *evsel)
476{
477 int i;
478 int err = 0;
479
480 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
481 err = perf_evsel__run_ioctl(evsel, ioc: PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx: i);
482 return err;
483}
484
485int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
486{
487 return perf_evsel__run_ioctl(evsel, ioc: PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx);
488}
489
490int perf_evsel__disable(struct perf_evsel *evsel)
491{
492 int i;
493 int err = 0;
494
495 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
496 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i);
497 return err;
498}
499
500int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter)
501{
502 int err = 0, i;
503
504 for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++)
505 err = perf_evsel__run_ioctl(evsel,
506 PERF_EVENT_IOC_SET_FILTER,
507 (void *)filter, i);
508 return err;
509}
510
511struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel)
512{
513 return evsel->cpus;
514}
515
516struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel)
517{
518 return evsel->threads;
519}
520
521struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel)
522{
523 return &evsel->attr;
524}
525
526int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
527{
528 if (ncpus == 0 || nthreads == 0)
529 return 0;
530
531 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
532 if (evsel->sample_id == NULL)
533 return -ENOMEM;
534
535 evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
536 if (evsel->id == NULL) {
537 xyarray__delete(evsel->sample_id);
538 evsel->sample_id = NULL;
539 return -ENOMEM;
540 }
541
542 return 0;
543}
544
545void perf_evsel__free_id(struct perf_evsel *evsel)
546{
547 struct perf_sample_id_period *pos, *n;
548
549 xyarray__delete(evsel->sample_id);
550 evsel->sample_id = NULL;
551 zfree(&evsel->id);
552 evsel->ids = 0;
553
554 perf_evsel_for_each_per_thread_period_safe(evsel, n, pos) {
555 list_del_init(entry: &pos->node);
556 free(pos);
557 }
558}
559
560bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel)
561{
562 return (evsel->attr.sample_type & PERF_SAMPLE_READ) &&
563 (evsel->attr.sample_type & PERF_SAMPLE_TID) &&
564 evsel->attr.inherit;
565}
566
567u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, bool per_thread)
568{
569 struct hlist_head *head;
570 struct perf_sample_id_period *res;
571 int hash;
572
573 if (!per_thread)
574 return &sid->period;
575
576 hash = hash_32(tid, PERF_SAMPLE_ID__HLIST_BITS);
577 head = &sid->periods[hash];
578
579 hlist_for_each_entry(res, head, hnode)
580 if (res->tid == tid)
581 return &res->period;
582
583 if (sid->evsel == NULL)
584 return NULL;
585
586 res = zalloc(sizeof(struct perf_sample_id_period));
587 if (res == NULL)
588 return NULL;
589
590 INIT_LIST_HEAD(list: &res->node);
591 res->tid = tid;
592
593 list_add_tail(new: &res->node, head: &sid->evsel->per_stream_periods);
594 hlist_add_head(n: &res->hnode, h: &sid->periods[hash]);
595
596 return &res->period;
597}
598
599void perf_counts_values__scale(struct perf_counts_values *count,
600 bool scale, __s8 *pscaled)
601{
602 s8 scaled = 0;
603
604 if (scale) {
605 if (count->run == 0) {
606 scaled = -1;
607 count->val = 0;
608 } else if (count->run < count->ena) {
609 scaled = 1;
610 count->val = (u64)((double)count->val * count->ena / count->run);
611 }
612 }
613
614 if (pscaled)
615 *pscaled = scaled;
616}
617

source code of linux/tools/lib/perf/evsel.c